From f0e944696144016ca59aaed02381f7ea9d1ef848 Mon Sep 17 00:00:00 2001 From: SoniEx2 Date: Tue, 4 Oct 2022 22:44:46 -0300 Subject: Initial VM work --- src/parser.rs | 223 ++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 145 insertions(+), 78 deletions(-) (limited to 'src/parser.rs') diff --git a/src/parser.rs b/src/parser.rs index c929653..97185ac 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -15,6 +15,8 @@ use crate::Predicate; use crate::errors::PatternError; use crate::vm::PatternConstants; use crate::vm::PatternElement; +use crate::vm::PatternToken; +use crate::vm::Type; /// try! with bools. (the b comes from bool.) macro_rules! bry { @@ -93,7 +95,7 @@ impl_trait! { O: Serialize, { fn start(value: &'r mut Parser<'s, PKey, OKey, O>) -> Self { - value.consts.protos.push(Default::default()); + value.tokens.push(Default::default()); Self { root: value, } @@ -101,9 +103,9 @@ impl_trait! { fn commit(self) -> usize { let mut self_ = ManuallyDrop::new(self); - let proto = self_.root.consts.protos.pop().unwrap(); + let proto = self_.root.tokens.pop().unwrap(); let id = self_.root.closed_subtrees.next().unwrap(); - self_.root.consts.protos.insert(id, proto); + self_.root.tokens.insert(id, proto); id } @@ -124,7 +126,7 @@ impl_trait! { impl trait Drop { fn drop(&mut self) { // remove "partial" proto - self.root.consts.protos.pop().expect("SubtreeHelper"); + self.root.tokens.pop().expect("SubtreeHelper"); } } } @@ -154,7 +156,7 @@ impl_trait! { O: Serialize, { fn start(value: &'r mut Parser<'s, PKey, OKey, O>) -> Self { - let len = value.consts.protos.last().unwrap().len(); + let len = value.tokens.last().unwrap().len(); Self { root: value, len : len, @@ -162,7 +164,7 @@ impl_trait! { } fn commit(self) { - std::mem::forget(self) + let _self = std::mem::ManuallyDrop::new(self); } impl trait std::ops::Deref { @@ -181,7 +183,7 @@ impl_trait! { impl trait Drop { fn drop(&mut self) { - let proto = self.root.consts.protos.last_mut().unwrap(); + let proto = self.root.tokens.last_mut().unwrap(); assert!(proto.len() >= self.len); while proto.len() > self.len { let _ = proto.pop(); @@ -203,6 +205,7 @@ where pred_ids: BTreeMap, obj_ids: BTreeMap, consts: PatternConstants, + tokens: Vec>, closed_subtrees: std::ops::RangeFrom, } @@ -215,6 +218,24 @@ where OKey: Borrow + Ord, O: Serialize, { + /// Creates a new `Parser`. + fn new( + base: &'s str, + preds: Option>>, + objs: Option>, + ) -> Self { + Self { + base: base, + preds: preds, + objs: objs, + tokens: Default::default(), + pred_ids: Default::default(), + obj_ids: Default::default(), + consts: Default::default(), + closed_subtrees: (0..), + } + } + /// str_literal <- sp ( ( "'" str_char* ( "'" / ( !. -> ErrorStrEnd ) ) ( '?' -> MarkSkippable ) ) -> String ) sp /// str_char <- ( str_escape / [^%'] ) /// str_escape <- '%' ( '%' / "'" ) / ( ( '%' .? ) -> ErrorStrEscape ) @@ -260,8 +281,8 @@ where self_.consts.strings.push(string); self_.consts.strings.len() - 1 }); - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::StringKey(id, skippable)); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::String(id, skippable)); *s = cursor; true })) @@ -318,14 +339,14 @@ where self_.consts.regices.push(re); self_.consts.regices.len() - 1 }); - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::RegexKey(id, skippable)); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::Regex(id, skippable)); *s = cursor; true })) } - /// matcher <- sp ( parameter / str_literal / re_literal / key_subtree ) sp + /// matcher <- sp ( parameter / str_literal / re_literal / predicate / ty ) sp fn matcher(&mut self, s: &mut &'s str) -> Result> { let mut cursor = *s; Ok(lblock!('matches: { @@ -334,7 +355,9 @@ where _ if self.parameter(&mut cursor)? => {}, _ if self.str_literal(&mut cursor)? => {}, _ if self.re_literal(&mut cursor)? => {}, - _ if self.key_subtree(&mut cursor)? => {}, + _ if self.predicate(&mut cursor)? => {}, + _ if self.ty(&mut cursor)? => {}, + //_ if self.key_subtree(&mut cursor)? => {}, _ => bry!('matches false), } self.sp(&mut cursor); @@ -343,7 +366,7 @@ where })) } - /// tag <- sp ( '->' -> Arrow ) sp ( matcher / name sp matcher? ) sp predicate* ( '' -> End ) sp + /// tag <- sp ( '->' -> Arrow ) sp key_subtree? sp ( matcher / name sp matcher? ) ( '' -> End ) sp fn tag(&mut self, s: &mut &'s str) -> Result> { let mut cursor = *s; Ok(lblock!('matches: { @@ -351,22 +374,21 @@ where bry!('matches strip_prefix(&mut cursor, "->")); let mut self_ = TagHelper::start(&mut *self); { - let proto = self_.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::Arrow); + let proto = self_.tokens.last_mut().expect("protos"); + proto.push(PatternToken::Arrow); } self_.sp(&mut cursor); + let _ = self_.key_subtree(&mut cursor)?; + self_.sp(&mut cursor); if !self_.matcher(&mut cursor)? { bry!('matches self_.name(&mut cursor)?); self_.sp(&mut cursor); - // NOTE: *optional* let _ = self_.matcher(&mut cursor)?; } self_.sp(&mut cursor); - while self_.predicate(&mut cursor)? { - } { - let proto = self_.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::End); + let proto = self_.tokens.last_mut().expect("protos"); + proto.push(PatternToken::End); } self_.commit(); *s = cursor; @@ -409,8 +431,8 @@ where self.consts.strings.push(name.into()); self.consts.strings.len() - 1 }); - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::Identifier(id)); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::Identifier(id)); self.sp(&mut cursor); *s = cursor; true @@ -443,22 +465,69 @@ where }, Ok, )?; - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::ParameterKey(id, skippable)); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::Parameter(id, skippable)); self.sp(&mut cursor); *s = cursor; true })) } - /// predicate <- sp ':' ( '?'? -> MarkSkippable ) '$' ( identifier -> Predicate ) sp + /// ty <- sp ':' ( '?'? -> MarkSkippable ) ( identifier -> Type ) sp + fn ty(&mut self, s: &mut &'s str) -> Result> { + let mut cursor = *s; + Ok(lblock!('matches: { + self.sp(&mut cursor); + bry!('matches strip_prefix(&mut cursor, ":")); + let custom = strip_prefix(&mut cursor, "$"); + let skippable = strip_prefix(&mut cursor, "?"); + let start = cursor; + bry!('matches self.identifier(&mut cursor)?); + let name = &start[..pos_of(start, cursor).unwrap_or(start.len())]; + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::Type(match name { + "bool" => Type::Bool, + "i8" => Type::I8, + "i16" => Type::I16, + "i32" => Type::I32, + "i64" => Type::I64, + "i128" => Type::I128, + "u8" => Type::U8, + "u16" => Type::U16, + "u32" => Type::U32, + "u64" => Type::U64, + "u128" => Type::U128, + "f32" => Type::F32, + "f64" => Type::F64, + "char" => Type::Char, + "str" => Type::Str, + "string" => Type::String, + "bytes" => Type::Bytes, + "bytebuf" => Type::ByteBuf, + "option" => Type::Option, + "unit" => Type::Unit, + "seq" => Type::Seq, + "map" => Type::Map, + //"tuple" => Type::Tuple(usize), + _ => { + let pos = pos_of(self.base, start).unwrap(); + return Err(PatternError::UnknownPredicate(pos, name)) + } + }, skippable)); + self.sp(&mut cursor); + *s = cursor; + true + })) + } + + /// predicate <- sp ':' '$' ( '?'? -> MarkSkippable ) ( identifier -> Predicate ) sp fn predicate(&mut self, s: &mut &'s str) -> Result> { let mut cursor = *s; Ok(lblock!('matches: { self.sp(&mut cursor); bry!('matches strip_prefix(&mut cursor, ":")); + let custom = strip_prefix(&mut cursor, "$"); let skippable = strip_prefix(&mut cursor, "?"); - bry!('matches strip_prefix(&mut cursor, "$")); let start = cursor; bry!('matches self.identifier(&mut cursor)?); let name = &start[..pos_of(start, cursor).unwrap_or(start.len())]; @@ -478,15 +547,17 @@ where }, Ok, )?; - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::ApplyPredicate(id, skippable)); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::ApplyPredicate(id, skippable)); self.sp(&mut cursor); *s = cursor; - true + let pos = pos_of(self.base, start).unwrap(); + return Err(PatternError::Unimplemented(pos, cursor)); + //true })) } - /// key_subtree <- sp '[' sp name? sp predicate* sp subtree sp ( ']' / unexpected_token / unexpected_end ) sp ( '?'? -> MarkSkippable ) + /// key_subtree <- sp '[' sp ( matcher / name sp matcher? ) sp subtree sp ( ']' / unexpected_token / unexpected_end ) sp ( '?'? -> MarkSkippable ) fn key_subtree(&mut self, s: &mut &'s str) -> Result> { let mut cursor = *s; Ok(lblock!('matches: { @@ -494,9 +565,10 @@ where bry!('matches strip_prefix(&mut cursor, "[")); self.sp(&mut cursor); let mut subtree = SubtreeHelper::start(&mut *self); - subtree.name(&mut cursor)?; - subtree.sp(&mut cursor); - while subtree.predicate(&mut cursor)? { + if !subtree.matcher(&mut cursor)? { + bry!('matches subtree.name(&mut cursor)?); + subtree.sp(&mut cursor); + let _ = subtree.matcher(&mut cursor)?; } subtree.sp(&mut cursor); bry!('matches subtree.subtree(&mut cursor)?); @@ -512,13 +584,13 @@ where let skippable = strip_prefix(&mut cursor, "?"); *s = cursor; let id = subtree.commit(); - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::KeySubtree(id, skippable)); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::KeySubtree(id, skippable)); true })) } - /// value_subtree <- sp '(' sp predicate* sp subtree sp ( ')' / unexpected_token / unexpected_end ) sp ( '?'? -> MarkSkippable ) + /// value_subtree <- sp '(' sp subtree sp ( ')' / unexpected_token / unexpected_end ) sp ( '?'? -> MarkSkippable ) fn value_subtree(&mut self, s: &mut &'s str) -> Result> { let mut cursor = *s; Ok(lblock!('matches: { @@ -526,9 +598,6 @@ where bry!('matches strip_prefix(&mut cursor, "(")); self.sp(&mut cursor); let mut subtree = SubtreeHelper::start(&mut *self); - while subtree.predicate(&mut cursor)? { - } - subtree.sp(&mut cursor); bry!('matches subtree.subtree(&mut cursor)?); subtree.sp(&mut cursor); bry!('matches @@ -542,8 +611,8 @@ where let skippable = strip_prefix(&mut cursor, "?"); *s = cursor; let id = subtree.commit(); - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::ValueSubtree(id, skippable)); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::ValueSubtree(id, skippable)); true })) } @@ -572,8 +641,8 @@ where } self.sp(&mut cursor); while self.value_subtree(&mut cursor)? { - let proto = self.consts.protos.last_mut().expect("protos"); - proto.push(PatternElement::End); + let proto = self.tokens.last_mut().expect("protos"); + proto.push(PatternToken::End); } self.sp(&mut cursor); *s = cursor; @@ -581,11 +650,12 @@ where Ok(true) } - /// pattern <- ( subtree / unexpected_token ) ( !. / unexpected_token ) + /// pattern <- ( matcher? sp subtree / unexpected_token ) ( !. / unexpected_token ) fn pattern(&mut self, s: &mut &'s str) -> Result> { let mut cursor = *s; Ok(lblock!('matches: { let mut subtree = SubtreeHelper::start(&mut *self); + let _ = subtree.matcher(&mut cursor)?; bry!('matches subtree.subtree(&mut cursor)? || @@ -616,22 +686,14 @@ where OKey: Borrow + Ord, O: Serialize, { - let mut parser = Parser::<'s, PKey, OKey, O> { - base: input, - preds: preds, - objs: objs, - pred_ids: Default::default(), - obj_ids: Default::default(), - consts: Default::default(), - closed_subtrees: (0..), - }; + let mut parser = Parser::new(input, preds, objs); let mut parsed = input; let matched = parser.pattern(&mut parsed)?; assert!(matched); assert_eq!(parsed, ""); - assert_eq!(parser.closed_subtrees.next().unwrap(), parser.consts.protos.len()); + assert_eq!(parser.closed_subtrees.next().unwrap(), parser.tokens.len()); Ok(parser.consts) } @@ -643,18 +705,20 @@ mod tests { use proptest::prelude::*; + fn prep_parser<'s>(s: &'s str) -> Parser<'s, &'static str, &'static str, ()> { + let mut parser = Parser::< + 's, &'static str, &'static str, () + >::new(s, None, None); + parser.tokens.push(Default::default()); + parser + } + #[test] fn test_identifier() { fn identifier_input<'s>(s: &mut &'s str) -> Result> { - let mut parser = Parser::<'s, &'static str, &'static str, ()> { - base: *s, - preds: None, - objs: None, - pred_ids: Default::default(), - obj_ids: Default::default(), - consts: Default::default(), - closed_subtrees: (0..), - }; + let mut parser = Parser::< + 's, &'static str, &'static str, () + >::new(s, None, None); parser.identifier(s) } for mut identifier in vec!["test", "Test", "_test", "_Test", "_123",] { @@ -672,20 +736,6 @@ mod tests { proptest! { #[test] fn test_no_crash(s in ".{0,4096}") { - fn prep_parser<'s>(s: &'s str) -> Parser<'s, &'static str, &'static str, ()> { - let mut parser = Parser::<'s, &'static str, &'static str, ()> { - base: s, - preds: None, - objs: None, - pred_ids: Default::default(), - obj_ids: Default::default(), - consts: Default::default(), - closed_subtrees: (0..), - }; - parser.consts.protos.push(Default::default()); - parser - } - let _ = prep_parser(&s).str_literal(&mut &*s); let _ = prep_parser(&s).re_literal(&mut &*s); let _ = prep_parser(&s).matcher(&mut &*s); @@ -695,6 +745,7 @@ mod tests { let _ = prep_parser(&s).name(&mut &*s); let _ = prep_parser(&s).parameter(&mut &*s); let _ = prep_parser(&s).predicate(&mut &*s); + let _ = prep_parser(&s).ty(&mut &*s); let _ = prep_parser(&s).key_subtree(&mut &*s); let _ = prep_parser(&s).value_subtree(&mut &*s); let _ = prep_parser(&s).unexpected_end(&mut &*s); @@ -703,5 +754,21 @@ mod tests { let _ = prep_parser(&s).pattern(&mut &*s); } } + + #[test] + fn test_pattern_tag() { + fn check_tag<'s>(s: &mut &'s str) -> ( + Result>, + Parser::< + 's, &'static str, &'static str, () + > + ) { + let mut parser = prep_parser(s); + let result = parser.tag(s); + (result, parser) + } + + // TODO + } } -- cgit 1.4.1