Merge pull request bytecodealliance#25 from Robbepop/try-to-fix-wasmparser-benchmarks-2

Try to fix benchmark tests - bytecodealliance#2

Author: yurydelendik

crates/wasmparser/benches/benchmark.rs

@@ -1,9 +1,9 @@
 #[macro_use]
 extern crate criterion;
 
-use criterion::Criterion;
-use std::fs::{read_dir, File};
-use std::io::Read;
+use criterion::{black_box, Criterion};
+use std::fs;
+use std::io;
 use std::path::Path;
 use std::path::PathBuf;
 use wasmparser::{
@@ -22,65 +22,208 @@ const VALIDATOR_CONFIG: Option<ValidatingParserConfig> = Some(ValidatingParserCo
     },
 });
 
-fn read_all_wasm<'a, T>(mut d: T)
+/// A benchmark input.
+pub struct BenchmarkInput {
+    /// The path to the benchmark file important for handling errors.
+    pub path: PathBuf,
+    /// The encoded Wasm module that is run by the benchmark.
+    pub wasm: Vec<u8>,
+}
+
+impl BenchmarkInput {
+    /// Creates a new benchmark input.
+    pub fn new(test_path: PathBuf, encoded_wasm: Vec<u8>) -> Self {
+        Self {
+            path: test_path,
+            wasm: encoded_wasm,
+        }
+    }
+}
+
+/// Read a `.wat` formatted benchmark test file as benchmark input.
+fn read_wat_module(path: &PathBuf) -> BenchmarkInput {
+    let encoded_wasm =
+        wat::parse_file(path).expect("encountered error while parsing `.wat` file into `.wasm`");
+    BenchmarkInput::new(path.clone(), encoded_wasm)
+}
+
+/// Read a `.wast` formatted benchmark test file as benchmark input.
+///
+/// We simply pull out all the module directives of the `.wast` file and return them.
+fn read_wast_module(path: &PathBuf) -> Vec<BenchmarkInput> {
+    let mut wast_file = fs::File::open(path)
+        .ok()
+        .expect("encountered error while reading `.wast` benchmark file");
+    let mut wast_file_contents = String::new();
+    use io::Read as _;
+    wast_file
+        .read_to_string(&mut wast_file_contents)
+        .expect("encountered error while reading `.wast` benchmark file to string");
+    let mut inputs = Vec::new();
+    if let Ok(parse_buffer) = wast::parser::ParseBuffer::new(&wast_file_contents) {
+        'outer: while let Ok(directive) = wast::parser::parse::<wast::WastDirective>(&parse_buffer)
+        {
+            match directive {
+                wast::WastDirective::Module(mut module) => {
+                    let encoded_wasm = module
+                        .encode()
+                        .expect("encountered error while encoding the Wast module into Wasm");
+                    inputs.push(BenchmarkInput::new(path.clone(), encoded_wasm));
+                }
+                _ => continue 'outer,
+            }
+        }
+    }
+    inputs
+}
+
+/// Visits all directory entries within the given directory path.
+///
+/// - `pred` can be used to filter some directories, e.g. all directories named
+///   `"proposals"`.
+/// - `cb` is the callback that is being called for every file within the non
+///   filtered and visited directories.
+fn visit_dirs<P, F>(dir: &Path, pred: &P, cb: &mut F) -> io::Result<()>
 where
-    T: WasmDecoder<'a>,
+    P: Fn(&fs::DirEntry) -> bool,
+    F: FnMut(&fs::DirEntry),
 {
-    loop {
-        match *d.read() {
-            ParserState::Error(ref e) => panic!("unexpected error while reading Wasm: {}", e),
-            ParserState::EndWasm => return,
-            _ => (),
+    if dir.is_dir() {
+        for entry in fs::read_dir(dir)? {
+            let entry = entry?;
+            let path = entry.path();
+            if path.is_dir() && pred(&entry) {
+                visit_dirs(&path, pred, cb)?;
+            } else {
+                cb(&entry);
+            }
         }
     }
+    Ok(())
 }
 
-fn read_file_data(path: &PathBuf) -> Vec<u8> {
-    let mut data = Vec::new();
-    let mut f = File::open(path).ok().unwrap();
-    f.read_to_end(&mut data).unwrap();
-    data
+/// Returns a vector of all found benchmark input files under the given directory.
+///
+/// Benchmark input files can be `.wat` or `.wast` formatted files.
+/// For `.wast` files we pull out all the module directives and run them in the benchmarks.
+fn collect_test_files<P>(path: P) -> Vec<BenchmarkInput>
+where
+    P: AsRef<Path>,
+{
+    let mut file_contents: Vec<BenchmarkInput> = vec![];
+    visit_dirs(
+        path.as_ref(),
+        &|_| true, // accept all benchmarks
+        &mut |dir_entry| {
+            let ext: Option<String> = dir_entry
+                .path()
+                .extension()
+                .and_then(|ext| ext.to_str().map(|str| str.to_string()));
+            match ext.as_ref().map(|string| string.as_str()) {
+                Some("wat") => file_contents.push(read_wat_module(&dir_entry.path())),
+                Some("wast") => {
+                    for wasm_module in read_wast_module(&dir_entry.path()) {
+                        file_contents.push(wasm_module)
+                    }
+                }
+                _ => (),
+            }
+        },
+    )
+    .expect("encountered error while reading test directory");
+    file_contents
 }
 
-fn collect_test_files<P>(path: P) -> Vec<Vec<u8>>
+/// Reads the input given the Wasm parser or validator.
+///
+/// The `path` specifies which benchmark input file we are currently operating on
+/// so that we can report better errors in case of failures.
+fn read_all_wasm<'a, T>(path: &PathBuf, mut d: T)
 where
-    P: AsRef<Path>,
+    T: WasmDecoder<'a>,
 {
-    let mut file_contents: Vec<Vec<u8>> = vec![];
-    for entry in read_dir(path).expect("cannot find the benchmark test files") {
-        let dir = entry.unwrap();
-        if !dir.file_type().unwrap().is_file() {
-            continue;
+    loop {
+        match *d.read() {
+            ParserState::Error(ref e) => {
+                panic!("unexpected error while reading Wasm from {:?}: {}", path, e)
+            }
+            ParserState::EndWasm => return,
+            _ => (),
         }
-        file_contents.push(read_file_data(&dir.path()));
     }
-    file_contents
+}
+
+/// Returns the default benchmark inputs that are proper `wasmparser` benchmark
+/// test inputs.
+fn collect_benchmark_inputs() -> Vec<BenchmarkInput> {
+    let from_testsuite = collect_test_files("../../testsuite");
+    let from_tests = collect_test_files("../../tests");
+    from_testsuite
+        .into_iter()
+        .chain(from_tests.into_iter())
+        .collect::<Vec<_>>()
 }
 
 fn it_works_benchmark(c: &mut Criterion) {
-    let mut data = collect_test_files("../../tests");
-    c.bench_function("it works benchmark", move |b| {
-        for d in &mut data {
-            b.iter(|| read_all_wasm(Parser::new(d.as_slice())));
+    let mut inputs = collect_benchmark_inputs();
+    // Filter out all benchmark inputs that fail to parse via `wasmparser`.
+    inputs.retain(|input| {
+        let mut parser = Parser::new(input.wasm.as_slice());
+        'outer: loop {
+            match parser.read() {
+                ParserState::Error(_) => break 'outer false,
+                ParserState::EndWasm => break 'outer true,
+                _ => continue,
+            }
         }
     });
+    c.bench_function("it works benchmark", move |b| {
+        b.iter(|| {
+            for input in &mut inputs {
+                let _ = black_box(read_all_wasm(
+                    &input.path,
+                    Parser::new(input.wasm.as_slice()),
+                ));
+            }
+        })
+    });
 }
 
 fn validator_not_fails_benchmark(c: &mut Criterion) {
-    let mut data = collect_test_files("../../tests");
-    c.bench_function("validator no fails benchmark", move |b| {
-        for d in &mut data {
-            b.iter(|| read_all_wasm(ValidatingParser::new(d.as_slice(), VALIDATOR_CONFIG)));
+    let mut inputs = collect_benchmark_inputs();
+    // Filter out all benchmark inputs that fail to validate via `wasmparser`.
+    inputs.retain(|input| {
+        let mut parser = ValidatingParser::new(input.wasm.as_slice(), VALIDATOR_CONFIG);
+        'outer: loop {
+            match parser.read() {
+                ParserState::Error(_) => break 'outer false,
+                ParserState::EndWasm => break 'outer true,
+                _ => continue,
+            }
         }
     });
+    c.bench_function("validator no fails benchmark", move |b| {
+        b.iter(|| {
+            for input in &mut inputs {
+                let _ = black_box(read_all_wasm(
+                    &input.path,
+                    ValidatingParser::new(input.wasm.as_slice(), VALIDATOR_CONFIG),
+                ));
+            }
+        });
+    });
 }
 
 fn validate_benchmark(c: &mut Criterion) {
-    let mut data = collect_test_files("../../tests");
+    let mut inputs = collect_benchmark_inputs();
+    // Filter out all benchmark inputs that fail to validate via `wasmparser`.
+    inputs.retain(|input| validate(input.wasm.as_slice(), VALIDATOR_CONFIG).is_ok());
     c.bench_function("validate benchmark", move |b| {
-        for d in &mut data {
-            b.iter(|| validate(&d, VALIDATOR_CONFIG));
-        }
+        b.iter(|| {
+            for input in &mut inputs {
+                let _ = black_box(validate(input.wasm.as_slice(), VALIDATOR_CONFIG));
+            }
+        })
     });
 }