diff --git a/pb-rs/Cargo.toml b/pb-rs/Cargo.toml index 83f17d5..513f0f8 100644 --- a/pb-rs/Cargo.toml +++ b/pb-rs/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" nom = "7" log = "0.4.4" clap = { version = "2.33.1", optional = true } -env_logger = { version = "0.7.1", optional = true } +env_logger = { version = "0.9.0", optional = true } [dev-dependencies] walkdir = "2.3.1" diff --git a/pb-rs/src/errors.rs b/pb-rs/src/errors.rs index ecb17e4..ba4347b 100644 --- a/pb-rs/src/errors.rs +++ b/pb-rs/src/errors.rs @@ -7,6 +7,8 @@ pub enum Error { Io(io::Error), /// Nom Error Nom(nom::Err>), + /// Nom's other failure case; giving up in the middle of a file + TrailingGarbage(String), /// No .proto file provided NoProto, /// Cannot read input file @@ -59,6 +61,7 @@ impl std::fmt::Display for Error { match self { Error::Io(e) => write!(f, "{}", e), Error::Nom(e) => write!(f, "{}", e), + Error::TrailingGarbage(s) => write!(f, "parsing abandoned near: {:?}", s), Error::NoProto => write!(f, "No .proto file provided"), Error::InputFile(file) => write!(f, "Cannot read input file '{}'", file), Error::OutputFile(file) => write!(f, "Cannot read output file '{}'", file), diff --git a/pb-rs/src/parser.rs b/pb-rs/src/parser.rs index 328af92..31af9ba 100644 --- a/pb-rs/src/parser.rs +++ b/pb-rs/src/parser.rs @@ -8,9 +8,9 @@ use crate::types::{ use nom::{ branch::alt, - bytes::complete::{tag, take_until}, + bytes::complete::{tag, take_until, take_while}, character::complete::{alphanumeric1, digit1, hex_digit1, line_ending, multispace1}, - combinator::{map, map_res, not, opt, recognize, value}, + combinator::{map, map_res, opt, recognize, value}, multi::{many0, many1, separated_list0, separated_list1}, sequence::{delimited, pair, preceded, separated_pair, terminated, tuple}, IResult, @@ -60,7 +60,7 @@ fn integer(input: &str) -> IResult<&str, i32> { fn comment(input: &str) -> IResult<&str, ()> { value( (), - tuple((tag("//"), many0(not(line_ending)), opt(line_ending))), + tuple((tag("//"), take_while(|c| !"\n\r".contains(c)), opt(line_ending))), )(input) } @@ -327,7 +327,7 @@ fn message(input: &str) -> IResult<&str, Message> { delimited( tag("{"), many0(delimited(many0(br), message_event, many0(br))), - tag("}"), + preceded(many0(br), tag("}")), ), ), opt(pair(many0(br), tag(";"))), @@ -370,7 +370,7 @@ fn enumerator(input: &str) -> IResult<&str, Enumerator> { delimited(pair(tag("enum"), many1(br)), word, many0(br)), delimited( pair(tag("{"), many0(br)), - separated_list0(br, enum_field), + separated_list0(many0(br), enum_field), pair(many0(br), tag("}")), ), ), @@ -422,6 +422,18 @@ mod test { use std::path::Path; + fn assert_complete(parse: nom::IResult<&str, T>) -> T { + let (rem, obj) = parse.expect("valid parse"); + assert_eq!(rem, "", "expected no trailing data"); + obj + } + + fn assert_desc(msg: &str) -> FileDescriptor { + let (rem, obj) = file_descriptor(msg).expect("valid parse"); + assert_eq!("", rem, "expected no trailing data"); + obj + } + #[test] fn test_message() { let msg = r#"message ReferenceData @@ -444,6 +456,13 @@ mod test { } } + #[test] + fn empty_message() { + let mess = assert_complete(message("message Vec { }")); + assert_eq!("Vec", mess.name); + assert_eq!(0, mess.fields.len()); + } + #[test] fn test_enum() { let msg = r#"enum PairingStatus { @@ -459,6 +478,11 @@ mod test { } } + #[test] + fn comments() { + assert_complete(comment("// foo\n")); + } + #[test] fn test_ignore() { let msg = r#"option optimize_for = SPEED;"#; @@ -487,6 +511,14 @@ mod test { ); } + #[test] + fn leading_comment() { + let msg = r#"//foo + message Bar {}"#; + let desc = assert_desc(msg); + assert_eq!(1, desc.messages.len()); + } + #[test] fn test_package() { let msg = r#" @@ -587,6 +619,18 @@ mod test { } } + #[test] + fn enum_comments() { + let msg = r#"enum Turn { + UP = 1; + // for what? + // for what, you ask? + DOWN = 2; + }"#; + let en = assert_complete(enumerator(msg)); + assert_eq!(2, en.fields.len()); + } + #[test] fn test_rpc_service() { let msg = r#" diff --git a/pb-rs/src/types.rs b/pb-rs/src/types.rs index 091100b..251131a 100644 --- a/pb-rs/src/types.rs +++ b/pb-rs/src/types.rs @@ -1926,7 +1926,11 @@ impl FileDescriptor { /// Opens a proto file, reads it and returns raw parsed data pub fn read_proto(in_file: &Path, import_search_path: &[PathBuf]) -> Result { let file = std::fs::read_to_string(in_file)?; - let (_, mut desc) = file_descriptor(&file).map_err(|e| Error::Nom(e))?; + let (rem, mut desc) = file_descriptor(&file).map_err(|e| Error::Nom(e))?; + let rem = rem.trim(); + if !rem.is_empty() { + return Err(Error::TrailingGarbage(rem.chars().take(50).collect())); + } for mut m in &mut desc.messages { if m.path.as_os_str().is_empty() { m.path = in_file.to_path_buf(); diff --git a/perftest/Cargo.toml b/perftest/Cargo.toml index d885048..61276bc 100644 --- a/perftest/Cargo.toml +++ b/perftest/Cargo.toml @@ -9,7 +9,8 @@ edition = "2021" [dependencies] protobuf = "2.25.2" quick-protobuf = { path = "../quick-protobuf" } -rand = "0.5.5" +rand = "0.8" +rand_xoshiro = "0.6" prost = "0.9.0" bytes = "1" @@ -18,3 +19,6 @@ protobuf-codegen-pure = "2.25.2" pb-rs = { path = "../pb-rs" } prost-build = "0.9.0" [features] + +[profile.release] +lto = true diff --git a/perftest/src/main.rs b/perftest/src/main.rs index 86d101a..7e6424e 100644 --- a/perftest/src/main.rs +++ b/perftest/src/main.rs @@ -1,4 +1,3 @@ -use rand::{rngs::SmallRng, Rng, SeedableRng}; use std::default::Default; use std::fmt::Debug; use std::fs::File; @@ -12,6 +11,8 @@ use perftest_data_quick::PerftestData as QuickPerftestData; use prost::Message as ProstMessage; use protobuf::Message; +use rand::{Rng, SeedableRng}; +use rand_xoshiro::Xoshiro256PlusPlus; use quick_protobuf::{BytesReader, MessageRead, MessageWrite, Reader, Writer}; mod perftest_data; @@ -20,8 +21,9 @@ mod perftest_data_quick { include!(concat!(env!("OUT_DIR"), "/perftest_data_quick.rs")); } -const SEED: [u8; 16] = [ +const SEED: [u8; 32] = [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, + 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, ]; fn measure R>(iter: u128, mut f: F, check: Option<&R>) -> u128 { @@ -41,12 +43,12 @@ impl TestRunner { fn run_test(&self, data: &[M]) -> [u128; 4] { let mut a = [0; 4]; - let mut rng = SmallRng::from_seed(SEED); + let mut rng = Xoshiro256PlusPlus::from_seed(SEED); let mut random_data = Vec::new(); let mut total_size = 0; while total_size < self.data_size { - let item = data[rng.gen_range(0, data.len())].clone(); + let item = data[rng.gen_range(0..data.len())].clone(); total_size += item.compute_size(); random_data.push(item); } @@ -117,12 +119,12 @@ impl TestRunner { { let mut b = [0; 4]; - let mut rng = SmallRng::from_seed(SEED); + let mut rng = Xoshiro256PlusPlus::from_seed(SEED); let mut random_data: Vec = Vec::new(); let mut total_size = 0; while total_size < self.data_size { - let ref item = data[rng.gen_range(0, data.len())]; + let ref item = data[rng.gen_range(0..data.len())]; random_data.push(item.clone()); total_size += item.get_size() as u32; } @@ -178,12 +180,12 @@ impl TestRunner { fn quick_run_test_strings(&self, data: &[perftest_data_quick::TestStrings]) -> [u128; 4] { let mut b = [0; 4]; - let mut rng = SmallRng::from_seed(SEED); + let mut rng = Xoshiro256PlusPlus::from_seed(SEED); let mut random_data = Vec::new(); let mut total_size = 0; while total_size < self.data_size { - let ref item = data[rng.gen_range(0, data.len())]; + let ref item = data[rng.gen_range(0..data.len())]; random_data.push(item.clone()); total_size += item.get_size() as u32; } @@ -230,12 +232,12 @@ impl TestRunner { fn quick_run_test_bytes(&self, data: &[perftest_data_quick::TestBytes]) -> [u128; 4] { let mut b = [0; 4]; - let mut rng = SmallRng::from_seed(SEED); + let mut rng = Xoshiro256PlusPlus::from_seed(SEED); let mut random_data = Vec::new(); let mut total_size = 0; while total_size < self.data_size { - let ref item = data[rng.gen_range(0, data.len())]; + let ref item = data[rng.gen_range(0..data.len())]; random_data.push(item.clone()); total_size += item.get_size() as u32; } @@ -285,12 +287,12 @@ impl TestRunner { ) -> [u128; 4] { let mut c = [0; 4]; - let mut rng = SmallRng::from_seed(SEED); + let mut rng = Xoshiro256PlusPlus::from_seed(SEED); let mut random_data: Vec = Vec::new(); let mut total_size = 0; while total_size < self.data_size { - let ref item = data[rng.gen_range(0, data.len())]; + let ref item = data[rng.gen_range(0..data.len())]; random_data.push(item.clone()); total_size += item.encoded_len() as u32; }