moonbeam_rpc_trace/

lib.rs

// Copyright 2019-2025 PureStake Inc.
// This file is part of Moonbeam.

// Moonbeam is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Moonbeam is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Moonbeam.  If not, see <http://www.gnu.org/licenses/>.

//! `trace_filter` RPC handler and its associated service task.
//! The RPC handler rely on `CacheTask` which provides a future that must be run inside a tokio
//! executor.
//!
//! The implementation is composed of multiple tasks :
//! - Many calls the RPC handler `Trace::filter`, communicating with the main task.
//! - A main `CacheTask` managing the cache and the communication between tasks.
//! - For each traced block an async task responsible to wait for a permit, spawn a blocking
//!   task and waiting for the result, then send it to the main `CacheTask`.

use futures::{select, stream::FuturesUnordered, FutureExt, StreamExt};
use std::{collections::BTreeMap, future::Future, marker::PhantomData, sync::Arc, time::Duration};
use tokio::{
    sync::{mpsc, oneshot, Semaphore},
    time::sleep,
};
use tracing::{instrument, Instrument};

use sc_client_api::backend::{Backend, StateBackend, StorageProvider};
use sc_utils::mpsc::TracingUnboundedSender;
use sp_api::{ApiExt, Core, ProvideRuntimeApi};
use sp_block_builder::BlockBuilder;
use sp_blockchain::{
    Backend as BlockchainBackend, Error as BlockChainError, HeaderBackend, HeaderMetadata,
};
use sp_runtime::traits::{BlakeTwo256, Block as BlockT, Header as HeaderT};
use substrate_prometheus_endpoint::{
    register, Counter, PrometheusError, Registry as PrometheusRegistry, U64,
};

use ethereum_types::H256;
use fc_storage::StorageOverride;
use fp_rpc::EthereumRuntimeRPCApi;

use moonbeam_client_evm_tracing::{
    formatters::ResponseFormatter,
    types::block::{self, TransactionTrace},
};
pub use moonbeam_rpc_core_trace::{FilterRequest, TraceServer};
use moonbeam_rpc_core_types::{RequestBlockId, RequestBlockTag};
use moonbeam_rpc_primitives_debug::DebugRuntimeApi;

type TxsTraceRes = Result<Vec<TransactionTrace>, String>;

/// RPC handler. Will communicate with a `CacheTask` through a `CacheRequester`.
pub struct Trace<B, C> {
    _phantom: PhantomData<B>,
    client: Arc<C>,
    requester: CacheRequester,
    max_count: u32,
}

impl<B, C> Clone for Trace<B, C> {
    fn clone(&self) -> Self {
        Self {
            _phantom: PhantomData,
            client: Arc::clone(&self.client),
            requester: self.requester.clone(),
            max_count: self.max_count,
        }
    }
}

impl<B, C> Trace<B, C>
where
    B: BlockT<Hash = H256> + Send + Sync + 'static,
    B::Header: HeaderT<Number = u32>,
    C: HeaderMetadata<B, Error = BlockChainError> + HeaderBackend<B>,
    C: Send + Sync + 'static,
{
    /// Create a new RPC handler.
    pub fn new(client: Arc<C>, requester: CacheRequester, max_count: u32) -> Self {
        Self {
            client,
            requester,
            max_count,
            _phantom: PhantomData,
        }
    }

    /// Convert an optional block ID (number or tag) to a block height.
    fn block_id(&self, id: Option<RequestBlockId>) -> Result<u32, &'static str> {
        match id {
            Some(RequestBlockId::Number(n)) => Ok(n),
            None | Some(RequestBlockId::Tag(RequestBlockTag::Latest)) => {
                Ok(self.client.info().best_number)
            }
            Some(RequestBlockId::Tag(RequestBlockTag::Earliest)) => Ok(0),
            Some(RequestBlockId::Tag(RequestBlockTag::Finalized)) => {
                Ok(self.client.info().finalized_number)
            }
            Some(RequestBlockId::Tag(RequestBlockTag::Pending)) => {
                Err("'pending' is not supported")
            }
            Some(RequestBlockId::Hash(_)) => Err("Block hash not supported"),
        }
    }

    /// `trace_filter` endpoint (wrapped in the trait implementation with futures compatibility)
    async fn filter(self, req: FilterRequest) -> TxsTraceRes {
        let from_block = self.block_id(req.from_block)?;
        let to_block = self.block_id(req.to_block)?;
        let block_heights = from_block..=to_block;

        let count = req.count.unwrap_or(self.max_count);
        if count > self.max_count {
            return Err(format!(
                "count ({}) can't be greater than maximum ({})",
                count, self.max_count
            ));
        }

        // Build a list of all the Substrate block hashes that need to be traced.
        let mut block_hashes = vec![];
        for block_height in block_heights {
            if block_height == 0 {
                continue; // no traces for genesis block.
            }

            let block_hash = self
                .client
                .hash(block_height)
                .map_err(|e| {
                    format!(
                        "Error when fetching block {} header : {:?}",
                        block_height, e
                    )
                })?
                .ok_or_else(|| format!("Block with height {} don't exist", block_height))?;

            block_hashes.push(block_hash);
        }

        // Start a batch with these blocks.
        let batch_id = self.requester.start_batch(block_hashes.clone()).await?;
        // Fetch all the traces. It is done in another function to simplify error handling and allow
        // to call the following `stop_batch` regardless of the result. This is important for the
        // cache cleanup to work properly.
        let res = self.fetch_traces(req, &block_hashes, count as usize).await;
        // Stop the batch, allowing the cache task to remove useless non-started block traces and
        // start the expiration delay.
        self.requester.stop_batch(batch_id).await;

        res
    }

    async fn fetch_traces(
        &self,
        req: FilterRequest,
        block_hashes: &[H256],
        count: usize,
    ) -> TxsTraceRes {
        let from_address = req.from_address.unwrap_or_default();
        let to_address = req.to_address.unwrap_or_default();

        let mut traces_amount: i64 = -(req.after.unwrap_or(0) as i64);
        let mut traces = vec![];

        for &block_hash in block_hashes {
            // Request the traces of this block to the cache service.
            // This will resolve quickly if the block is already cached, or wait until the block
            // has finished tracing.
            let block_traces = self.requester.get_traces(block_hash).await?;

            // Filter addresses.
            let mut block_traces: Vec<_> = block_traces
                .iter()
                .filter(|trace| match trace.action {
                    block::TransactionTraceAction::Call { from, to, .. } => {
                        (from_address.is_empty() || from_address.contains(&from))
                            && (to_address.is_empty() || to_address.contains(&to))
                    }
                    block::TransactionTraceAction::Create { from, .. } => {
                        (from_address.is_empty() || from_address.contains(&from))
                            && to_address.is_empty()
                    }
                    block::TransactionTraceAction::Suicide { address, .. } => {
                        (from_address.is_empty() || from_address.contains(&address))
                            && to_address.is_empty()
                    }
                })
                .cloned()
                .collect();

            // Don't insert anything if we're still before "after"
            traces_amount += block_traces.len() as i64;
            if traces_amount > 0 {
                let traces_amount = traces_amount as usize;
                // If the current Vec of traces is across the "after" marker,
                // we skip some elements of it.
                if traces_amount < block_traces.len() {
                    let skip = block_traces.len() - traces_amount;
                    block_traces = block_traces.into_iter().skip(skip).collect();
                }

                traces.append(&mut block_traces);

                // If we go over "count" (the limit), we trim and exit the loop,
                // unless we used the default maximum, in which case we return an error.
                if traces_amount >= count {
                    if req.count.is_none() {
                        return Err(format!(
                            "the amount of traces goes over the maximum ({}), please use 'after' \
							and 'count' in your request",
                            self.max_count
                        ));
                    }

                    traces = traces.into_iter().take(count).collect();
                    break;
                }
            }
        }

        Ok(traces)
    }
}

#[jsonrpsee::core::async_trait]
impl<B, C> TraceServer for Trace<B, C>
where
    B: BlockT<Hash = H256> + Send + Sync + 'static,
    B::Header: HeaderT<Number = u32>,
    C: HeaderMetadata<B, Error = BlockChainError> + HeaderBackend<B>,
    C: Send + Sync + 'static,
{
    async fn filter(
        &self,
        filter: FilterRequest,
    ) -> jsonrpsee::core::RpcResult<Vec<TransactionTrace>> {
        self.clone()
            .filter(filter)
            .await
            .map_err(fc_rpc::internal_err)
    }
}

/// An opaque batch ID.
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct CacheBatchId(u64);

/// Requests the cache task can accept.
enum CacheRequest {
    /// Request to start caching the provided range of blocks.
    /// The task will add to blocks to its pool and immediately return a new batch ID.
    StartBatch {
        /// Returns the ID of the batch for cancellation.
        sender: oneshot::Sender<CacheBatchId>,
        /// List of block hash to trace.
        blocks: Vec<H256>,
    },
    /// Fetch the traces for given block hash.
    /// The task will answer only when it has processed this block.
    GetTraces {
        /// Returns the array of traces or an error.
        sender: oneshot::Sender<TxsTraceRes>,
        /// Hash of the block.
        block: H256,
    },
    /// Notify the cache that it can stop the batch with that ID. Any block contained only in
    /// this batch and still not started will be discarded.
    StopBatch { batch_id: CacheBatchId },
}

/// Allows to interact with the cache task.
#[derive(Clone)]
pub struct CacheRequester(TracingUnboundedSender<CacheRequest>);

impl CacheRequester {
    /// Request to start caching the provided range of blocks.
    /// The task will add to blocks to its pool and immediately return the batch ID.
    #[instrument(skip(self))]
    pub async fn start_batch(&self, blocks: Vec<H256>) -> Result<CacheBatchId, String> {
        let (response_tx, response_rx) = oneshot::channel();
        let sender = self.0.clone();

        sender
            .unbounded_send(CacheRequest::StartBatch {
                sender: response_tx,
                blocks,
            })
            .map_err(|e| {
                format!(
                    "Failed to send request to the trace cache task. Error : {:?}",
                    e
                )
            })?;

        response_rx.await.map_err(|e| {
            format!(
                "Trace cache task closed the response channel. Error : {:?}",
                e
            )
        })
    }

    /// Fetch the traces for given block hash.
    /// The task will answer only when it has processed this block.
    /// The block should be part of a batch first. If no batch has requested the block it will
    /// return an error.
    #[instrument(skip(self))]
    pub async fn get_traces(&self, block: H256) -> TxsTraceRes {
        let (response_tx, response_rx) = oneshot::channel();
        let sender = self.0.clone();

        sender
            .unbounded_send(CacheRequest::GetTraces {
                sender: response_tx,
                block,
            })
            .map_err(|e| {
                format!(
                    "Failed to send request to the trace cache task. Error : {:?}",
                    e
                )
            })?;

        response_rx
            .await
            .map_err(|e| {
                format!(
                    "Trace cache task closed the response channel. Error : {:?}",
                    e
                )
            })?
            .map_err(|e| format!("Failed to replay block. Error : {:?}", e))
    }

    /// Notify the cache that it can stop the batch with that ID. Any block contained only in
    /// this batch and still in the waiting pool will be discarded.
    #[instrument(skip(self))]
    pub async fn stop_batch(&self, batch_id: CacheBatchId) {
        let sender = self.0.clone();

        // Here we don't care if the request has been accepted or refused, the caller can't
        // do anything with it.
        let _ = sender
            .unbounded_send(CacheRequest::StopBatch { batch_id })
            .map_err(|e| {
                format!(
                    "Failed to send request to the trace cache task. Error : {:?}",
                    e
                )
            });
    }
}

/// Data stored for each block in the cache.
/// `active_batch_count` represents the number of batches using this
/// block. It will increase immediately when a batch is created, but will be
/// decrease only after the batch ends and its expiration delay passes.
/// It allows to keep the data in the cache for following requests that would use
/// this block, which is important to handle pagination efficiently.
struct CacheBlock {
    active_batch_count: usize,
    state: CacheBlockState,
}

/// State of a cached block. It can either be polled to be traced or cached.
enum CacheBlockState {
    /// Block has been added to the pool blocks to be replayed.
    /// It may be currently waiting to be replayed or being replayed.
    Pooled {
        started: bool,
        /// Multiple requests might query the same block while it is pooled to be
        /// traced. They response channel is stored here, and the result will be
        /// sent in all of them when the tracing is finished.
        waiting_requests: Vec<oneshot::Sender<TxsTraceRes>>,
        /// Channel used to unqueue a tracing that has not yet started.
        /// A tracing will be unqueued if it has not yet been started and the last batch
        /// needing this block is ended (ignoring the expiration delay).
        /// It is not used directly, but dropping will wake up the receiver.
        #[allow(dead_code)]
        unqueue_sender: oneshot::Sender<()>,
    },
    /// Tracing has been completed and the result is available. No Runtime API call
    /// will be needed until this block cache is removed.
    Cached { traces: TxsTraceRes },
}

/// Tracing a block is done in a separate tokio blocking task to avoid clogging the async threads.
/// For this reason a channel using this type is used by the blocking task to communicate with the
/// main cache task.
enum BlockingTaskMessage {
    /// Notify the tracing for this block has started as the blocking task got a permit from
    /// the semaphore. This is used to prevent the deletion of a cache entry for a block that has
    /// started being traced.
    Started { block_hash: H256 },
    /// The tracing is finished and the result is sent to the main task.
    Finished {
        block_hash: H256,
        result: TxsTraceRes,
    },
}

/// Type wrapper for the cache task, generic over the Client, Block and Backend types.
pub struct CacheTask<B, C, BE> {
    client: Arc<C>,
    backend: Arc<BE>,
    blocking_permits: Arc<Semaphore>,
    cached_blocks: BTreeMap<H256, CacheBlock>,
    batches: BTreeMap<u64, Vec<H256>>,
    next_batch_id: u64,
    metrics: Option<Metrics>,
    _phantom: PhantomData<B>,
}

impl<B, C, BE> CacheTask<B, C, BE>
where
    BE: Backend<B> + 'static,
    BE::State: StateBackend<BlakeTwo256>,
    C: ProvideRuntimeApi<B>,
    C: StorageProvider<B, BE>,
    C: HeaderMetadata<B, Error = BlockChainError> + HeaderBackend<B>,
    C: Send + Sync + 'static,
    B: BlockT<Hash = H256> + Send + Sync + 'static,
    B::Header: HeaderT<Number = u32>,
    C::Api: BlockBuilder<B>,
    C::Api: DebugRuntimeApi<B>,
    C::Api: EthereumRuntimeRPCApi<B>,
    C::Api: ApiExt<B>,
{
    /// Create a new cache task.
    ///
    /// Returns a Future that needs to be added to a tokio executor, and a handle allowing to
    /// send requests to the task.
    pub fn create(
        client: Arc<C>,
        backend: Arc<BE>,
        cache_duration: Duration,
        blocking_permits: Arc<Semaphore>,
        overrides: Arc<dyn StorageOverride<B>>,
        prometheus: Option<PrometheusRegistry>,
    ) -> (impl Future<Output = ()>, CacheRequester) {
        // Communication with the outside world :
        let (requester_tx, mut requester_rx) =
            sc_utils::mpsc::tracing_unbounded("trace-filter-cache", 100_000);

        // Task running in the service.
        let task = async move {
			// The following variables are polled by the select! macro, and thus cannot be
			// part of Self without introducing borrowing issues.
			let mut batch_expirations = FuturesUnordered::new();
			let (blocking_tx, mut blocking_rx) =
				mpsc::channel(blocking_permits.available_permits() * 2);
			let metrics = if let Some(registry) = prometheus {
				match Metrics::register(&registry) {
					Ok(metrics) => Some(metrics),
					Err(err) => {
						log::error!(target: "tracing", "Failed to register metrics {err:?}");
						None
					}
				}
			} else {
				None
			};
			// Contains the inner state of the cache task, excluding the pooled futures/channels.
			// Having this object allows to refactor each event into its own function, simplifying
			// the main loop.
			let mut inner = Self {
				client,
				backend,
				blocking_permits,
				cached_blocks: BTreeMap::new(),
				batches: BTreeMap::new(),
				next_batch_id: 0,
				metrics,
				_phantom: Default::default(),
			};

			// Main event loop. This loop must not contain any direct .await, as we want to
			// react to events as fast as possible.
			loop {
				select! {
					request = requester_rx.next() => {
						match request {
							None => break,
							Some(CacheRequest::StartBatch {sender, blocks})
								=> inner.request_start_batch(&blocking_tx, sender, blocks, overrides.clone()),
							Some(CacheRequest::GetTraces {sender, block})
								=> inner.request_get_traces(sender, block),
							Some(CacheRequest::StopBatch {batch_id}) => {
								// Cannot be refactored inside `request_stop_batch` because
								// it has an unnamable type :C
								batch_expirations.push(async move {
									sleep(cache_duration).await;
									batch_id
								});

								inner.request_stop_batch(batch_id);
							},
						}
					},
					message = blocking_rx.recv().fuse() => {
						match message {
							None => (),
							Some(BlockingTaskMessage::Started { block_hash })
								=> inner.blocking_started(block_hash),
							Some(BlockingTaskMessage::Finished { block_hash, result })
								=> inner.blocking_finished(block_hash, result),
						}
					},
					batch_id = batch_expirations.next() => {
						match batch_id {
							None => (),
							Some(batch_id) => inner.expired_batch(batch_id),
						}
					}
				}
			}
		}
		.instrument(tracing::debug_span!("trace_filter_cache"));

        (task, CacheRequester(requester_tx))
    }

    /// Handle the creation of a batch.
    /// Will start the tracing process for blocks that are not already in the cache.
    #[instrument(skip(self, blocking_tx, sender, blocks, overrides))]
    fn request_start_batch(
        &mut self,
        blocking_tx: &mpsc::Sender<BlockingTaskMessage>,
        sender: oneshot::Sender<CacheBatchId>,
        blocks: Vec<H256>,
        overrides: Arc<dyn StorageOverride<B>>,
    ) {
        tracing::trace!("Starting batch {}", self.next_batch_id);
        self.batches.insert(self.next_batch_id, blocks.clone());

        for block in blocks {
            // The block is already in the cache, awesome!
            if let Some(block_cache) = self.cached_blocks.get_mut(&block) {
                block_cache.active_batch_count += 1;
                tracing::trace!(
                    "Cache hit for block {}, now used by {} batches.",
                    block,
                    block_cache.active_batch_count
                );
            }
            // Otherwise we need to queue this block for tracing.
            else {
                tracing::trace!("Cache miss for block {}, pooling it for tracing.", block);

                let blocking_permits = Arc::clone(&self.blocking_permits);
                let (unqueue_sender, unqueue_receiver) = oneshot::channel();
                let client = Arc::clone(&self.client);
                let backend = Arc::clone(&self.backend);
                let blocking_tx = blocking_tx.clone();
                let overrides = overrides.clone();

                // Spawn all block caching asynchronously.
                // It will wait to obtain a permit, then spawn a blocking task.
                // When the blocking task returns its result, it is sent
                // thought a channel to the main task loop.
                tokio::spawn(
                    async move {
                        tracing::trace!("Waiting for blocking permit or task cancellation");
                        let _permit = select!(
                            _ = unqueue_receiver.fuse() => {
                            tracing::trace!("Tracing of the block has been cancelled.");
                                return;
                            },
                            permit = blocking_permits.acquire().fuse() => permit,
                        );

                        // Warn the main task that block tracing as started, and
                        // this block cache entry should not be removed.
                        let _ = blocking_tx
                            .send(BlockingTaskMessage::Started { block_hash: block })
                            .await;

                        tracing::trace!("Start block tracing in a blocking task.");

                        // Perform block tracing in a tokio blocking task.
                        let result = async {
                            tokio::task::spawn_blocking(move || {
                                Self::cache_block(client, backend, block, overrides.clone())
                            })
                            .await
                            .map_err(|e| {
                                format!("Tracing Substrate block {} panicked : {:?}", block, e)
                            })?
                        }
                        .await
                        .map_err(|e| e.to_string());

                        tracing::trace!("Block tracing finished, sending result to main task.");

                        // Send a response to the main task.
                        let _ = blocking_tx
                            .send(BlockingTaskMessage::Finished {
                                block_hash: block,
                                result,
                            })
                            .await;
                    }
                    .instrument(tracing::trace_span!("Block tracing", block = %block)),
                );

                // Insert the block in the cache.
                self.cached_blocks.insert(
                    block,
                    CacheBlock {
                        active_batch_count: 1,
                        state: CacheBlockState::Pooled {
                            started: false,
                            waiting_requests: vec![],
                            unqueue_sender,
                        },
                    },
                );
            }
        }

        // Respond with the batch ID.
        let _ = sender.send(CacheBatchId(self.next_batch_id));

        // Increase batch ID for the next request.
        self.next_batch_id = self.next_batch_id.overflowing_add(1).0;
    }

    /// Handle a request to get the traces of the provided block.
    /// - If the result is stored in the cache, it sends it immediately.
    /// - If the block is currently being pooled, it is added to this block cache waiting list,
    ///   and all requests concerning this block will be satisfied when the tracing for this block
    ///   is finished.
    /// - If this block is missing from the cache, it means no batch asked for it. All requested
    ///   blocks should be contained in a batch beforehand, and thus an error is returned.
    #[instrument(skip(self))]
    fn request_get_traces(&mut self, sender: oneshot::Sender<TxsTraceRes>, block: H256) {
        if let Some(block_cache) = self.cached_blocks.get_mut(&block) {
            match &mut block_cache.state {
                CacheBlockState::Pooled {
                    ref mut waiting_requests,
                    ..
                } => {
                    tracing::warn!(
                        "A request asked a pooled block ({}), adding it to the list of \
						waiting requests.",
                        block
                    );
                    waiting_requests.push(sender);
                    if let Some(metrics) = &self.metrics {
                        metrics.tracing_cache_misses.inc();
                    }
                }
                CacheBlockState::Cached { traces, .. } => {
                    tracing::warn!(
                        "A request asked a cached block ({}), sending the traces directly.",
                        block
                    );
                    let _ = sender.send(traces.clone());
                    if let Some(metrics) = &self.metrics {
                        metrics.tracing_cache_hits.inc();
                    }
                }
            }
        } else {
            tracing::warn!(
                "An RPC request asked to get a block ({}) which was not batched.",
                block
            );
            let _ = sender.send(Err(format!(
                "RPC request asked a block ({}) that was not batched",
                block
            )));
        }
    }

    /// Handle a request to stop a batch.
    /// For all blocks that needed to be traced, are only in this batch and not yet started, their
    /// tracing is cancelled to save CPU-time and avoid attacks requesting large amount of blocks.
    /// This batch data is not yet removed however. Instead a expiration delay timer is started
    /// after which the data will indeed be cleared. (the code for that is in the main loop code
    /// as it involved an unnamable type :C)
    #[instrument(skip(self))]
    fn request_stop_batch(&mut self, batch_id: CacheBatchId) {
        tracing::trace!("Stopping batch {}", batch_id.0);
        if let Some(blocks) = self.batches.get(&batch_id.0) {
            for block in blocks {
                let mut remove = false;

                // We remove early the block cache if this batch is the last
                // pooling this block.
                if let Some(block_cache) = self.cached_blocks.get_mut(block) {
                    if block_cache.active_batch_count == 1
                        && matches!(
                            block_cache.state,
                            CacheBlockState::Pooled { started: false, .. }
                        )
                    {
                        remove = true;
                    }
                }

                if remove {
                    tracing::trace!("Pooled block {} is no longer requested.", block);
                    // Remove block from the cache. Drops the value,
                    // closing all the channels contained in it.
                    let _ = self.cached_blocks.remove(block);
                }
            }
        }
    }

    /// A tracing blocking task notifies it got a permit and is starting the tracing.
    /// This started status is stored to avoid removing this block entry.
    #[instrument(skip(self))]
    fn blocking_started(&mut self, block_hash: H256) {
        if let Some(block_cache) = self.cached_blocks.get_mut(&block_hash) {
            if let CacheBlockState::Pooled {
                ref mut started, ..
            } = block_cache.state
            {
                *started = true;
            }
        }
    }

    /// A tracing blocking task notifies it has finished the tracing and provide the result.
    #[instrument(skip(self, result))]
    fn blocking_finished(&mut self, block_hash: H256, result: TxsTraceRes) {
        // In some cases it might be possible to receive traces of a block
        // that has no entry in the cache because it was removed of the pool
        // and received a permit concurrently. We just ignore it.
        //
        // TODO : Should we add it back ? Should it have an active_batch_count
        // of 1 then ?
        if let Some(block_cache) = self.cached_blocks.get_mut(&block_hash) {
            if let CacheBlockState::Pooled {
                ref mut waiting_requests,
                ..
            } = block_cache.state
            {
                tracing::trace!(
                    "A new block ({}) has been traced, adding it to the cache and responding to \
					{} waiting requests.",
                    block_hash,
                    waiting_requests.len()
                );
                // Send result in waiting channels
                while let Some(channel) = waiting_requests.pop() {
                    let _ = channel.send(result.clone());
                }

                // Update cache entry
                block_cache.state = CacheBlockState::Cached { traces: result };
            }
        }
    }

    /// A batch expiration delay timer has completed. It performs the cache cleaning for blocks
    /// not longer used by other batches.
    #[instrument(skip(self))]
    fn expired_batch(&mut self, batch_id: CacheBatchId) {
        if let Some(batch) = self.batches.remove(&batch_id.0) {
            for block in batch {
                // For each block of the batch, we remove it if it was the
                // last batch containing it.
                let mut remove = false;
                if let Some(block_cache) = self.cached_blocks.get_mut(&block) {
                    block_cache.active_batch_count -= 1;

                    if block_cache.active_batch_count == 0 {
                        remove = true;
                    }
                }

                if remove {
                    let _ = self.cached_blocks.remove(&block);
                }
            }
        }
    }

    /// (In blocking task) Use the Runtime API to trace the block.
    #[instrument(skip(client, backend, overrides))]
    fn cache_block(
        client: Arc<C>,
        backend: Arc<BE>,
        substrate_hash: H256,
        overrides: Arc<dyn StorageOverride<B>>,
    ) -> TxsTraceRes {
        // Get Substrate block data.
        let api = client.runtime_api();
        let block_header = client
            .header(substrate_hash)
            .map_err(|e| {
                format!(
                    "Error when fetching substrate block {} header : {:?}",
                    substrate_hash, e
                )
            })?
            .ok_or_else(|| format!("Substrate block {} don't exist", substrate_hash))?;

        let height = *block_header.number();
        let substrate_parent_hash = *block_header.parent_hash();

        // Get Ethereum block data.
        let (eth_block, eth_transactions) = match (
            overrides.current_block(substrate_hash),
            overrides.current_transaction_statuses(substrate_hash),
        ) {
            (Some(a), Some(b)) => (a, b),
            _ => {
                return Err(format!(
                    "Failed to get Ethereum block data for Substrate block {}",
                    substrate_hash
                ))
            }
        };

        let eth_block_hash = eth_block.header.hash();
        let eth_tx_hashes = eth_transactions
            .iter()
            .map(|t| t.transaction_hash)
            .collect();

        // Get extrinsics (containing Ethereum ones)
        let extrinsics = backend
            .blockchain()
            .body(substrate_hash)
            .map_err(|e| {
                format!(
                    "Blockchain error when fetching extrinsics of block {} : {:?}",
                    height, e
                )
            })?
            .ok_or_else(|| format!("Could not find block {} when fetching extrinsics.", height))?;

        // Get DebugRuntimeApi version
        let trace_api_version = if let Ok(Some(api_version)) =
            api.api_version::<dyn DebugRuntimeApi<B>>(substrate_parent_hash)
        {
            api_version
        } else {
            return Err("Runtime api version call failed (trace)".to_string());
        };

        // Trace the block.
        let f = || -> Result<_, String> {
            let result = if trace_api_version >= 5 {
                api.trace_block(
                    substrate_parent_hash,
                    extrinsics,
                    eth_tx_hashes,
                    &block_header,
                )
            } else {
                // Get core runtime api version
                let core_api_version = if let Ok(Some(api_version)) =
                    api.api_version::<dyn Core<B>>(substrate_parent_hash)
                {
                    api_version
                } else {
                    return Err("Runtime api version call failed (core)".to_string());
                };

                // Initialize block: calls the "on_initialize" hook on every pallet
                // in AllPalletsWithSystem
                // This was fine before pallet-message-queue because the XCM messages
                // were processed by the "setValidationData" inherent call and not on an
                // "on_initialize" hook, which runs before enabling XCM tracing
                if core_api_version >= 5 {
                    api.initialize_block(substrate_parent_hash, &block_header)
                        .map_err(|e| format!("Runtime api access error: {:?}", e))?;
                } else {
                    #[allow(deprecated)]
                    api.initialize_block_before_version_5(substrate_parent_hash, &block_header)
                        .map_err(|e| format!("Runtime api access error: {:?}", e))?;
                }

                #[allow(deprecated)]
                api.trace_block_before_version_5(substrate_parent_hash, extrinsics, eth_tx_hashes)
            };

            result
                .map_err(|e| format!("Blockchain error when replaying block {} : {:?}", height, e))?
                .map_err(|e| {
                    tracing::warn!(
                        target: "tracing",
                        "Internal runtime error when replaying block {} : {:?}",
                        height,
                        e
                    );
                    format!(
                        "Internal runtime error when replaying block {} : {:?}",
                        height, e
                    )
                })?;

            Ok(moonbeam_rpc_primitives_debug::Response::Block)
        };

        let eth_transactions_by_index: BTreeMap<u32, H256> = eth_transactions
            .iter()
            .map(|t| (t.transaction_index, t.transaction_hash))
            .collect();

        let mut proxy = moonbeam_client_evm_tracing::listeners::CallList::default();
        proxy.using(f)?;

        let traces: Vec<TransactionTrace> =
			moonbeam_client_evm_tracing::formatters::TraceFilter::format(proxy)
				.ok_or("Fail to format proxy")?
				.into_iter()
				.filter_map(|mut trace| {
					match eth_transactions_by_index.get(&trace.transaction_position) {
						Some(transaction_hash) => {
							trace.block_hash = eth_block_hash;
							trace.block_number = height;
							trace.transaction_hash = *transaction_hash;

							// Reformat error messages.
							if let block::TransactionTraceOutput::Error(ref mut error) =
								trace.output
							{
								if error.as_slice() == b"execution reverted" {
									*error = b"Reverted".to_vec();
								}
							}

							Some(trace)
						}
						None => {
							log::warn!(
								target: "tracing",
								"A trace in block {} does not map to any known ethereum transaction. Trace: {:?}",
								height,
								trace,
							);
							None
						}
					}
				})
				.collect();

        Ok(traces)
    }
}

/// Prometheus metrics for tracing.
#[derive(Clone)]
pub(crate) struct Metrics {
    tracing_cache_hits: Counter<U64>,
    tracing_cache_misses: Counter<U64>,
}

impl Metrics {
    pub(crate) fn register(registry: &PrometheusRegistry) -> Result<Self, PrometheusError> {
        Ok(Self {
            tracing_cache_hits: register(
                Counter::new("tracing_cache_hits", "Number of tracing cache hits.")?,
                registry,
            )?,
            tracing_cache_misses: register(
                Counter::new("tracing_cache_misses", "Number of tracing cache misses.")?,
                registry,
            )?,
        })
    }
}