Finish most of Packer and Parser

This is still broken even tho it passes the tests

1 files changed, 52 insertions, 36 deletions

diff --git a/src/parser.rs b/src/parser.rs
index 595f157..0698b6b 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -144,6 +144,10 @@ impl_trait! {
             }
         }
 
+        fn is_empty(&self) -> bool {
+            self.root.consts.protos.last().unwrap().is_empty()
+        }
+
         fn commit(self) -> usize {
             let mut self_ = ManuallyDrop::new(self);
             let proto = self_.root.consts.protos.pop().unwrap();
@@ -207,11 +211,11 @@ impl_trait! {
         }
 
         fn commit(self) {
-            let _self = &mut *std::mem::ManuallyDrop::new(self);
+            let self_ = &mut *std::mem::ManuallyDrop::new(self);
             // we could write a proper parser for the token stream.
             //
             // we could also just do this instead.
-            match _self.root.tokens.drain(_self.len..).as_slice() {
+            match self_.root.tokens.drain(self_.len..).as_slice() {
                 &[
                     PatternToken::Arrow,
                     PatternToken::KeySubtree(index),
@@ -221,9 +225,9 @@ impl_trait! {
                     let tag = PatternElement::Tag {
                         key_subtree: Some(index),
                     };
-                    _self.root.consts.protos.last_mut().unwrap().push(tag);
+                    self_.root.consts.protos.last_mut().unwrap().push(tag);
                     let value = collect_name_and_value(name_value);
-                    _self.root.consts.protos.last_mut().unwrap().push(value);
+                    self_.root.consts.protos.last_mut().unwrap().push(value);
                 },
                 &[
                     PatternToken::Arrow,
@@ -233,9 +237,9 @@ impl_trait! {
                     let tag = PatternElement::Tag {
                         key_subtree: None,
                     };
-                    _self.root.consts.protos.last_mut().unwrap().push(tag);
+                    self_.root.consts.protos.last_mut().unwrap().push(tag);
                     let value = collect_name_and_value(name_value);
-                    _self.root.consts.protos.last_mut().unwrap().push(value);
+                    self_.root.consts.protos.last_mut().unwrap().push(value);
                 },
                 other => {
                     unreachable!("{other:?}");
@@ -259,9 +263,8 @@ impl_trait! {
 
         impl trait Drop {
             fn drop(&mut self) {
-                let proto = &mut self.root.tokens;
-                assert!(proto.len() >= self.len);
-                proto.drain(self.len..);
+                assert!(self.root.tokens.len() >= self.len);
+                self.root.tokens.drain(self.len..);
             }
         }
     }
@@ -355,8 +358,7 @@ where
                 self_.consts.strings.push(string);
                 self_.consts.strings.len() - 1
             });
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::String(id, skippable));
+            self.tokens.push(PatternToken::String(id, skippable));
             *s = cursor;
             true
         }))
@@ -413,8 +415,7 @@ where
                 self_.consts.regices.push(re);
                 self_.consts.regices.len() - 1
             });
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::Regex(id, skippable));
+            self.tokens.push(PatternToken::Regex(id, skippable));
             *s = cursor;
             true
         }))
@@ -447,8 +448,7 @@ where
             bry!('matches strip_prefix(&mut cursor, "->"));
             let mut self_ = TagHelper::start(&mut *self);
             {
-                let proto = &mut self_.tokens;
-                proto.push(PatternToken::Arrow);
+                self_.tokens.push(PatternToken::Arrow);
             }
             self_.sp(&mut cursor);
             let _ = self_.key_subtree(&mut cursor)?;
@@ -460,8 +460,7 @@ where
             }
             self_.sp(&mut cursor);
             {
-                let proto = &mut self_.tokens;
-                proto.push(PatternToken::End);
+                self_.tokens.push(PatternToken::End);
             }
             self_.commit();
             *s = cursor;
@@ -504,8 +503,7 @@ where
                 self.consts.strings.push(name.into());
                 self.consts.strings.len() - 1
             });
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::Identifier(id));
+            self.tokens.push(PatternToken::Identifier(id));
             self.sp(&mut cursor);
             *s = cursor;
             true
@@ -538,8 +536,7 @@ where
                 },
                 Ok,
             )?;
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::Parameter(id, skippable));
+            self.tokens.push(PatternToken::Parameter(id, skippable));
             self.sp(&mut cursor);
             *s = cursor;
             true
@@ -557,8 +554,9 @@ where
             let start = cursor;
             bry!('matches self.identifier(&mut cursor)?);
             let name = &start[..pos_of(start, cursor).unwrap_or(start.len())];
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::Type(match name {
+            self.tokens.push(PatternToken::Type(match name {
+                "any" => Type::Any,
+                "ignored_any" => Type::IgnoredAny,
                 "bool" => Type::Bool,
                 "i8" => Type::I8,
                 "i16" => Type::I16,
@@ -620,8 +618,7 @@ where
                 },
                 Ok,
             )?;
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::ApplyPredicate(id, skippable));
+            self.tokens.push(PatternToken::ApplyPredicate(id, skippable));
             self.sp(&mut cursor);
             *s = cursor;
             let pos = pos_of(self.base, start).unwrap();
@@ -639,7 +636,7 @@ where
             bry!('matches strip_prefix(&mut cursor, "["));
             self.sp(&mut cursor);
             let mut subtree = SubtreeHelper::start(&mut *self);
-            // FIXME handle `?`
+            // FIXME we may want to clean up tokens if these fail
             let marker = subtree.tokens.len();
             if !subtree.matcher(&mut cursor)? {
                 bry!('matches subtree.name(&mut cursor)?);
@@ -664,8 +661,7 @@ where
             //let skippable = strip_prefix(&mut cursor, "?");
             *s = cursor;
             let id = subtree.commit();
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::KeySubtree(id));
+            self.tokens.push(PatternToken::KeySubtree(id));
             true
         }))
     }
@@ -688,11 +684,12 @@ where
                 subtree.unexpected_end(&mut cursor)?
             );
             subtree.sp(&mut cursor);
-            let skippable = strip_prefix(&mut cursor, "?");
+            let optional = strip_prefix(&mut cursor, "?");
             *s = cursor;
-            let id = subtree.commit();
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::ValueSubtree(id, skippable));
+            if !subtree.is_empty() {
+                let id = subtree.commit();
+                self.tokens.push(PatternToken::ValueSubtree(id, optional));
+            }
             true
         }))
     }
@@ -720,9 +717,26 @@ where
         while self.tag(&mut cursor)? {
         }
         self.sp(&mut cursor);
+        let mut has_subtrees = false;
+        let marker = self.tokens.len();
+        self.consts.protos.last_mut().unwrap().push({
+            PatternElement::SubtreeMarker
+        });
         while self.value_subtree(&mut cursor)? {
-            let proto = &mut self.tokens;
-            proto.push(PatternToken::End);
+            let subtree = match self.tokens.drain(marker..).as_slice() {
+                &[] => continue,
+                &[PatternToken::ValueSubtree(index, optional)] => {
+                    PatternElement::ValueSubtree { index, optional }
+                },
+                other => {
+                    unreachable!("{other:?}")
+                }
+            };
+            self.consts.protos.last_mut().unwrap().push(subtree);
+        }
+        let proto = self.consts.protos.last_mut().unwrap();
+        if let Some(PatternElement::SubtreeMarker) = proto.last() {
+            proto.pop();
         }
         self.sp(&mut cursor);
         *s = cursor;
@@ -735,7 +749,7 @@ where
         let mut cursor = *s;
         Ok(lblock!('matches: {
             let mut subtree = SubtreeHelper::start(&mut *self);
-            // FIXME handle `?`
+            // FIXME we may want to clean up tokens if subtree.matcher errors
             let marker = subtree.tokens.len();
             let _ = subtree.matcher(&mut cursor)?;
             let value = match subtree.tokens.drain(marker..).as_slice() {
@@ -845,7 +859,8 @@ mod tests {
             let _ = prep_parser(&s).value_subtree(&mut &*s);
             let _ = prep_parser(&s).unexpected_end(&mut &*s);
             let _ = prep_parser(&s).unexpected_token(&mut &*s);
-            let _ = prep_parser(&s).subtree(&mut &*s);
+            // NOTE: never call subtree directly!!!
+            //let _ = prep_parser(&s).subtree(&mut &*s);
             let _ = prep_parser(&s).pattern(&mut &*s);
         }
     }
@@ -861,6 +876,7 @@ mod tests {
         run_pattern(":map");
         run_pattern(":?map");
         run_pattern(":map->[:str]:str");
+        run_pattern(":map(->[:str]:str)()");
     }
 }