Skip to content

Commit c047ec9

Browse files
skunertmelekesdavxy
authored
Add benchmarks for for import/validation/production (paritytech#2579)
* Add benchmarks * Remove endowed accounts * Fix group * Remove duplicate code * Comments * Use execute_block instead of block import * Apply suggestions from code review Co-authored-by: Anton <anton.kalyaev@gmail.com> * Add missing comments * Clippy * Reorganize utils * Remove unused import * Switch to compiled again * Apply suggestions from code review Co-authored-by: Davide Galassi <davxy@datawok.net> * Refactor local_testnet * Do not write wasm blob to file * Remove unnecessary block * Add comment fixes * fmt * lock --------- Co-authored-by: Anton <anton.kalyaev@gmail.com> Co-authored-by: Davide Galassi <davxy@datawok.net>
1 parent 005b9d0 commit c047ec9

14 files changed

Lines changed: 1709 additions & 39 deletions

cumulus/Cargo.lock

Lines changed: 524 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

cumulus/test/client/src/lib.rs

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ use sc_executor::{HeapAllocStrategy, WasmExecutionMethod, WasmExecutor};
2626
use sc_executor_common::runtime_blob::RuntimeBlob;
2727
use sc_service::client;
2828
use sp_blockchain::HeaderBackend;
29-
use sp_core::storage::Storage;
29+
use sp_core::{sr25519, storage::Storage, Pair};
3030
use sp_io::TestExternalities;
3131
use sp_runtime::{generic::Era, BuildStorage, SaturatedConversion};
3232

@@ -77,11 +77,22 @@ pub type Client = client::Client<Backend, Executor, Block, runtime::RuntimeApi>;
7777

7878
/// Parameters of test-client builder with test-runtime.
7979
#[derive(Default)]
80-
pub struct GenesisParameters;
80+
pub struct GenesisParameters {
81+
pub endowed_accounts: Vec<cumulus_test_runtime::AccountId>,
82+
}
8183

8284
impl substrate_test_client::GenesisInit for GenesisParameters {
8385
fn genesis_storage(&self) -> Storage {
84-
genesis_config().build_storage().unwrap()
86+
if self.endowed_accounts.is_empty() {
87+
genesis_config().build_storage().unwrap()
88+
} else {
89+
cumulus_test_service::testnet_genesis(
90+
cumulus_test_service::get_account_id_from_seed::<sr25519::Public>("Alice"),
91+
self.endowed_accounts.clone(),
92+
)
93+
.build_storage()
94+
.unwrap()
95+
}
8596
}
8697
}
8798

@@ -115,19 +126,26 @@ impl DefaultTestClientBuilderExt for TestClientBuilder {
115126
}
116127

117128
fn genesis_config() -> GenesisConfig {
118-
cumulus_test_service::local_testnet_genesis()
129+
cumulus_test_service::testnet_genesis_with_default_endowed(Default::default())
119130
}
120131

121-
/// Generate an extrinsic from the provided function call, origin and [`Client`].
122-
pub fn generate_extrinsic(
132+
/// Create an unsigned extrinsic from a runtime call.
133+
pub fn generate_unsigned(function: impl Into<RuntimeCall>) -> UncheckedExtrinsic {
134+
UncheckedExtrinsic::new_unsigned(function.into())
135+
}
136+
137+
/// Create a signed extrinsic from a runtime call and sign
138+
/// with the given key pair.
139+
pub fn generate_extrinsic_with_pair(
123140
client: &Client,
124-
origin: sp_keyring::AccountKeyring,
141+
origin: sp_core::sr25519::Pair,
125142
function: impl Into<RuntimeCall>,
143+
nonce: Option<u32>,
126144
) -> UncheckedExtrinsic {
127145
let current_block_hash = client.info().best_hash;
128146
let current_block = client.info().best_number.saturated_into();
129147
let genesis_block = client.hash(0).unwrap().unwrap();
130-
let nonce = 0;
148+
let nonce = nonce.unwrap_or_default();
131149
let period =
132150
BlockHashCount::get().checked_next_power_of_two().map(|c| c / 2).unwrap_or(2) as u64;
133151
let tip = 0;
@@ -158,6 +176,15 @@ pub fn generate_extrinsic(
158176
)
159177
}
160178

179+
/// Generate an extrinsic from the provided function call, origin and [`Client`].
180+
pub fn generate_extrinsic(
181+
client: &Client,
182+
origin: sp_keyring::AccountKeyring,
183+
function: impl Into<RuntimeCall>,
184+
) -> UncheckedExtrinsic {
185+
generate_extrinsic_with_pair(client, origin.into(), function, None)
186+
}
187+
161188
/// Transfer some token from one account to another using a provided test [`Client`].
162189
pub fn transfer(
163190
client: &Client,

cumulus/test/runtime/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ frame-system-rpc-runtime-api = { git = "https://github.com/paritytech/substrate"
1616
pallet-balances = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
1717
pallet-sudo = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
1818
pallet-timestamp = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
19+
pallet-glutton = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
1920
pallet-transaction-payment = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
2021
sp-api = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
2122
sp-block-builder = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }

cumulus/test/runtime/src/lib.rs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,12 @@ pub use frame_support::{
5757
},
5858
StorageValue,
5959
};
60-
use frame_system::limits::{BlockLength, BlockWeights};
60+
use frame_system::{
61+
limits::{BlockLength, BlockWeights},
62+
EnsureRoot,
63+
};
6164
pub use pallet_balances::Call as BalancesCall;
65+
pub use pallet_glutton::Call as GluttonCall;
6266
pub use pallet_sudo::Call as SudoCall;
6367
pub use pallet_timestamp::Call as TimestampCall;
6468
#[cfg(any(feature = "std", test))]
@@ -265,6 +269,12 @@ impl pallet_sudo::Config for Runtime {
265269
type WeightInfo = pallet_sudo::weights::SubstrateWeight<Runtime>;
266270
}
267271

272+
impl pallet_glutton::Config for Runtime {
273+
type RuntimeEvent = RuntimeEvent;
274+
type AdminOrigin = EnsureRoot<AccountId>;
275+
type WeightInfo = pallet_glutton::weights::SubstrateWeight<Runtime>;
276+
}
277+
268278
impl cumulus_pallet_parachain_system::Config for Runtime {
269279
type SelfParaId = ParachainId;
270280
type RuntimeEvent = RuntimeEvent;
@@ -296,6 +306,7 @@ construct_runtime! {
296306
Sudo: pallet_sudo,
297307
TransactionPayment: pallet_transaction_payment,
298308
TestPallet: test_pallet,
309+
Glutton: pallet_glutton,
299310
}
300311
}
301312

cumulus/test/service/Cargo.toml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ serde = { version = "1.0.163", features = ["derive"] }
1919
tokio = { version = "1.28.2", features = ["macros"] }
2020
tracing = "0.1.37"
2121
url = "2.3.1"
22+
tempfile = "3.5.0"
2223

2324
# Substrate
2425
frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -39,14 +40,19 @@ sp-arithmetic = { git = "https://github.com/paritytech/substrate", branch = "mas
3940
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
4041
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
4142
sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
43+
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
4244
sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
4345
sp-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
4446
sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" }
4547
sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
4648
sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
4749
sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" }
50+
sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
4851
substrate-test-client = { git = "https://github.com/paritytech/substrate", branch = "master" }
4952
sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
53+
sc-block-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
54+
sc-executor-wasmtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
55+
sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "master" }
5056

5157
# Polkadot
5258
polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "master" }
@@ -70,10 +76,21 @@ cumulus-test-relay-validation-worker-provider = { path = "../relay-validation-wo
7076
cumulus-test-runtime = { path = "../runtime" }
7177
cumulus-relay-chain-minimal-node = { path = "../../client/relay-chain-minimal-node" }
7278
cumulus-client-pov-recovery = { path = "../../client/pov-recovery" }
79+
cumulus-test-relay-sproof-builder = { path = "../relay-sproof-builder" }
80+
cumulus-pallet-parachain-system = { path = "../../pallets/parachain-system", default-features = false }
81+
pallet-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
7382

7483
[dev-dependencies]
7584
futures = "0.3.28"
7685
portpicker = "0.1.1"
86+
kitchensink-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
87+
rococo-parachain-runtime = { path = "../../parachains/runtimes/testing/rococo-parachain" }
88+
node-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
89+
pallet-im-online = { git = "https://github.com/paritytech/substrate", branch = "master" }
90+
node-primitives = { git = "https://github.com/paritytech/substrate", branch = "master" }
91+
sp-consensus-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master" }
92+
sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" }
93+
cumulus-test-client = { path = "../client" }
7794

7895
# Polkadot dependencies
7996
polkadot-test-service = { git = "https://github.com/paritytech/polkadot", branch = "master" }
@@ -88,3 +105,27 @@ runtime-benchmarks = ["polkadot-test-service/runtime-benchmarks"]
88105
[[bench]]
89106
name = "transaction_throughput"
90107
harness = false
108+
109+
[[bench]]
110+
name = "block_import"
111+
harness = false
112+
113+
[[bench]]
114+
name = "block_production"
115+
harness = false
116+
117+
[[bench]]
118+
name = "block_production_glutton"
119+
harness = false
120+
121+
[[bench]]
122+
name = "block_import_glutton"
123+
harness = false
124+
125+
[[bench]]
126+
name = "validate_block"
127+
harness = false
128+
129+
[[bench]]
130+
name = "validate_block_glutton"
131+
harness = false
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// This file is part of Cumulus.
2+
3+
// Copyright (C) Parity Technologies (UK) Ltd.
4+
// SPDX-License-Identifier: Apache-2.0
5+
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
18+
use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
19+
20+
use sc_client_api::UsageProvider;
21+
22+
use core::time::Duration;
23+
use cumulus_primitives_core::ParaId;
24+
25+
use sc_block_builder::{BlockBuilderProvider, RecordProof};
26+
use sp_api::{Core, ProvideRuntimeApi};
27+
use sp_keyring::Sr25519Keyring::Alice;
28+
29+
use cumulus_test_service::bench_utils as utils;
30+
31+
fn benchmark_block_import(c: &mut Criterion) {
32+
sp_tracing::try_init_simple();
33+
34+
let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed");
35+
let para_id = ParaId::from(100);
36+
let tokio_handle = runtime.handle();
37+
38+
// Create enough accounts to fill the block with transactions.
39+
// Each account should only be included in one transfer.
40+
let (src_accounts, dst_accounts, account_ids) = utils::create_benchmark_accounts();
41+
42+
let alice = runtime.block_on(
43+
cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Alice)
44+
// Preload all accounts with funds for the transfers
45+
.endowed_accounts(account_ids)
46+
.build(),
47+
);
48+
49+
let client = alice.client;
50+
51+
let (max_transfer_count, extrinsics) =
52+
utils::create_benchmarking_transfer_extrinsics(&client, &src_accounts, &dst_accounts);
53+
54+
let parent_hash = client.usage_info().chain.best_hash;
55+
let mut block_builder =
56+
client.new_block_at(parent_hash, Default::default(), RecordProof::No).unwrap();
57+
for extrinsic in extrinsics {
58+
block_builder.push(extrinsic).unwrap();
59+
}
60+
let benchmark_block = block_builder.build().unwrap();
61+
62+
let mut group = c.benchmark_group("Block import");
63+
group.sample_size(20);
64+
group.measurement_time(Duration::from_secs(120));
65+
group.throughput(Throughput::Elements(max_transfer_count as u64));
66+
67+
group.bench_function(format!("(transfers = {}) block import", max_transfer_count), |b| {
68+
b.iter_batched(
69+
|| benchmark_block.block.clone(),
70+
|block| {
71+
client.runtime_api().execute_block(parent_hash, block).unwrap();
72+
},
73+
BatchSize::SmallInput,
74+
)
75+
});
76+
}
77+
78+
criterion_group!(benches, benchmark_block_import);
79+
criterion_main!(benches);
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// This file is part of Cumulus.
2+
3+
// Copyright (C) Parity Technologies (UK) Ltd.
4+
// SPDX-License-Identifier: Apache-2.0
5+
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
18+
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
19+
20+
use sc_client_api::UsageProvider;
21+
use sp_api::{Core, ProvideRuntimeApi};
22+
use sp_arithmetic::Perbill;
23+
24+
use core::time::Duration;
25+
use cumulus_primitives_core::ParaId;
26+
27+
use sc_block_builder::{BlockBuilderProvider, RecordProof};
28+
use sp_keyring::Sr25519Keyring::Alice;
29+
30+
use cumulus_test_service::bench_utils as utils;
31+
32+
fn benchmark_block_import(c: &mut Criterion) {
33+
sp_tracing::try_init_simple();
34+
35+
let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed");
36+
let para_id = ParaId::from(100);
37+
let tokio_handle = runtime.handle();
38+
39+
let alice = runtime.block_on(
40+
cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Alice).build(),
41+
);
42+
let client = alice.client;
43+
44+
let mut group = c.benchmark_group("Block import");
45+
group.sample_size(20);
46+
group.measurement_time(Duration::from_secs(120));
47+
48+
let mut initialize_glutton_pallet = true;
49+
for (compute_percent, storage_percent) in &[
50+
(Perbill::from_percent(100), Perbill::from_percent(0)),
51+
(Perbill::from_percent(100), Perbill::from_percent(100)),
52+
] {
53+
let block = utils::set_glutton_parameters(
54+
&client,
55+
initialize_glutton_pallet,
56+
compute_percent,
57+
storage_percent,
58+
);
59+
initialize_glutton_pallet = false;
60+
61+
runtime.block_on(utils::import_block(&client, &block, false));
62+
63+
// Build the block we will use for benchmarking
64+
let parent_hash = client.usage_info().chain.best_hash;
65+
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
66+
let mut block_builder =
67+
client.new_block_at(parent_hash, Default::default(), RecordProof::No).unwrap();
68+
block_builder
69+
.push(utils::extrinsic_set_validation_data(parent_header.clone()).clone())
70+
.unwrap();
71+
block_builder.push(utils::extrinsic_set_time(&client)).unwrap();
72+
let benchmark_block = block_builder.build().unwrap();
73+
74+
group.bench_function(
75+
format!(
76+
"(compute = {:?}, storage = {:?}) block import",
77+
compute_percent, storage_percent
78+
),
79+
|b| {
80+
b.iter_batched(
81+
|| benchmark_block.block.clone(),
82+
|block| {
83+
client.runtime_api().execute_block(parent_hash, block).unwrap();
84+
},
85+
BatchSize::SmallInput,
86+
)
87+
},
88+
);
89+
}
90+
}
91+
92+
criterion_group!(benches, benchmark_block_import);
93+
criterion_main!(benches);

0 commit comments

Comments
 (0)