Release first v0.12 alpha

.github/workflows/rust.yml

@@ -1,6 +1,8 @@
 name: Rust
 
-on: [push]
+on: 
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   build:

CONTRIBUTING.md

@@ -7,14 +7,14 @@ Please note that [ressa](https://github.com/freemasen/ressa) and [resast](https:
 
 I do not work on this full time, please be patient if I am not able to respond quickly.
 
-The primary development branch is the `next` branch. It would be ideal to create any pull requests against that branch over `master` or one of the other feature branches that might have been missed when cleaning up.
-
 For any PRs know that the code must pass ci tests before they will be reviewed/merged. These test include the following commands you could use to check your version.
-```sh
-$ npm i
-$ cargo test
-$ cargo run --example major_libs
+
+```shell
+npm i
+cargo test
+cargo run --example major_libs
 ```
+
 The release flag in the above is due to the fact that this example is a naive benchmark to validate that changes haven't completely ruined the performance. Feel free to leave this flag off when you are testing for a PR.
 
 This will run all of the project's unit tests as well as a test against some major js libraries, namely [Angular-js](angularjs.org), [Jquery](jquery.com), [React/React-Dom](reactjs.org), [Vue](vuejs.org), [Moment.js](momentjs.com) and [Dexie](dexie.org).
@@ -63,6 +63,7 @@ The overall code layout works like this.
     - `is_other_whitesapce`: the ECMA spec says that any Zs category character is valid whitespace. This function will test any exotic whitespaces 
 
 # Testing
+
 There are a few sets of JavaScript files that are required to run the tests in this repository. The first set can be easily aquired by running `npm install` in the root of this project. An additional test is also available behind a feature flag `moz_central` that requires the JIT Test files from the FireFox repository, the expectation is that these will exist in the folder `moz-central` in the root of this project. To get these files you can either manually download and unzip them by following [this link](https://hg.mozilla.org/mozilla-central/archive/tip.zip/js/src/jit-test/tests/) or you can execute the following command.
 
 ```sh
@@ -74,4 +75,4 @@ To run these tests simple execute the following command.
 
 ```sh
 cargo test --features moz_central -- moz_central
-```
+```

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ress"
-version = "0.11.7"
+version = "0.12.0-alpha.1"
 authors = ["Robert Masen <r.f.masen@gmail.com>"]
 description = "A scanner/tokenizer for JS files"
 keywords = ["JavaScript", "parsing", "JS", "ES", "ECMA"]

examples/major_libs/src/main.rs

@@ -12,6 +12,7 @@ use std::{
     time::{Duration, SystemTime},
 };
 
+#[derive(Debug, Default, Clone, Copy, PartialEq)]
 struct Args {
     pub angular: bool,
     pub jquery: bool,
@@ -22,29 +23,19 @@ struct Args {
     pub dexie: bool,
 }
 
-impl ::std::default::Default for Args {
-    fn default() -> Args {
-        Args {
-            angular: false,
-            jquery: false,
-            react: false,
-            react_dom: false,
-            vue: false,
-            moment: false,
-            dexie: false,
-        }
-    }
-}
-
 impl Args {
     fn pristine(&self) -> bool {
-        !self.angular
-            && !self.jquery
-            && !self.react
-            && !self.react_dom
-            && !self.vue
-            && !self.moment
-            && !self.dexie
+        self == &Self::default()
+    }
+
+    fn mark_all_true(&mut self) {
+        self.angular = true;
+        self.jquery = true;
+        self.react = true;
+        self.react_dom = true;
+        self.vue = true;
+        self.moment = true;
+        self.dexie = true;
     }
 }
 
@@ -70,6 +61,9 @@ fn main() {
             a.dexie = true;
         }
     }
+    if a.pristine() {
+        a.mark_all_true();
+    }
     if a.jquery {
         jquery();
     }
@@ -91,15 +85,6 @@ fn main() {
     if a.dexie {
         dexie();
     }
-    if a.pristine() {
-        jquery();
-        angular1();
-        react();
-        react_dom();
-        vue();
-        moment();
-        dexie();
-    }
 }
 
 fn jquery() {

src/error.rs

@@ -27,3 +27,30 @@ impl ::std::fmt::Display for RawError {
         write!(f, "{} at {}", self.msg, self.idx)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn display() {
+        assert_eq!(
+            Error {
+                line: 1,
+                column: 1,
+                msg: "err".to_string(),
+                idx: 0,
+            }
+            .to_string(),
+            "err at 1:1"
+        );
+        assert_eq!(
+            RawError {
+                msg: "err".to_string(),
+                idx: 0,
+            }
+            .to_string(),
+            "err at 0"
+        );
+    }
+}

src/lib.rs

@@ -141,6 +141,7 @@ impl<T> Item<T> {
             location,
         }
     }
+
     fn new_(
         token: Token<T>,
         span_start: usize,
@@ -262,7 +263,7 @@ impl<'b> Scanner<'b> {
                 return Some(Err(e));
             }
         };
-
+        debug!("get_next_token\n{:?}\n{:?}", self.last_three, next.token);
         let ret = if next.token.is_div_punct() && self.is_regex_start() {
             self.manual_scanner.next_regex(next.span.len())?
         } else {
@@ -454,6 +455,7 @@ impl<'b> Scanner<'b> {
     ///
     /// > used in determining if we are at a regex or not
     fn check_for_expression(token: MetaToken) -> bool {
+        debug!("check_for_expression {:?}", token);
         if Self::is_op(token) {
             true
         } else {
@@ -710,7 +712,7 @@ this.y = 0;
     fn validate(s: Scanner, expected: Vec<Token<&str>>) {
         for (i, (lhs, rhs)) in s.zip(expected.into_iter()).enumerate() {
             let lhs = lhs.unwrap();
-            println!("{:?}, {:?}", lhs.token, rhs);
+            debug!("{:?}, {:?}", lhs.token, rhs);
             assert_eq!((i, lhs.token), (i, rhs));
         }
     }
@@ -755,6 +757,7 @@ this.y = 0;
         let r = s.next().unwrap().unwrap();
         assert_eq!(r.token, Token::RegEx(regex));
     }
+
     #[test]
     fn regex_replace() {
         let expect = vec![
@@ -837,6 +840,7 @@ f`;
         assert_eq!(format!("{}", Position::new(1, 25)), "1:25".to_string(),);
         assert_eq!(format!("{}", Position::new(25, 0)), "25:0".to_string(),);
     }
+
     #[test]
     fn position_ord() {
         assert!(
@@ -933,4 +937,81 @@ ley z = 9;";
         let re = s.next().unwrap().unwrap();
         assert!(re.token.is_regex(), "regex was not a regex: {:?}", re);
     }
+
+    #[test]
+    fn is_helpers() {
+        assert!(!Item::new(
+            Token::Ident(Ident::from("ident")),
+            Span::new(0, 1),
+            SourceLocation::new(Position::new(1, 1), Position::new(1, 2))
+        )
+        .is_string());
+        assert!(Item::new(
+            Token::String(StringLit::double("ident", false)),
+            Span::new(0, 1),
+            SourceLocation::new(Position::new(1, 1), Position::new(1, 2))
+        )
+        .is_string());
+        assert!(!Item::new(
+            Token::String(StringLit::double("ident", false)),
+            Span::new(0, 1),
+            SourceLocation::new(Position::new(1, 1), Position::new(1, 2))
+        )
+        .is_template());
+        assert!(Item::new(
+            Token::Template(Template::no_sub_template("ident", false, false, false)),
+            Span::new(0, 1),
+            SourceLocation::new(Position::new(1, 1), Position::new(1, 2))
+        )
+        .is_template());
+    }
+
+    #[test]
+    #[ignore = "regex detection is broken... should fix that..."]
+    fn function_division() {
+        let div2 = "let b = function() {} / 100;";
+        run_regex_test(div2, false);
+    }
+
+    #[test]
+    #[ignore = "regex detection is broken... should fix that..."]
+    fn regex() {
+        let div1 = "let a = 0 / 1";
+        let div2 = "let b = function() {} / 100;";
+        let re1 = "let c = /.+/";
+        let re2 = "let d = /asdf/g";
+        run_regex_test(div1, false);
+        run_regex_test(div2, false);
+        run_regex_test(re1, true);
+        run_regex_test(re2, true);
+    }
+
+    #[track_caller]
+    fn run_regex_test(js: &str, should_include: bool) {
+        for item in Scanner::new(js) {
+            match item {
+                Ok(item) => {
+                    if !should_include && item.token.is_regex() {
+                        let leading = item.span.start.saturating_sub(1);
+                        let region = item.span.end - item.span.start;
+                        let trailing = js.len() - (leading + region);
+                        panic!(
+                            "Unexpected regex:\n`{}`\n{}{}{}",
+                            js,
+                            " ".repeat(leading),
+                            "^".repeat(region),
+                            " ".repeat(trailing),
+                        )
+                    }
+                }
+                Err(e) => {
+                    let mut cursor = " ".repeat(js.len());
+                    if let Some(cursor) = unsafe { cursor.as_bytes_mut() }.get_mut(e.idx) {
+                        *cursor = b'^';
+                    }
+                    panic!("Invalid JS: {}\n`{}`\n{}", e.msg, js, cursor,);
+                }
+            }
+        }
+    }
 }

src/look_behind.rs

@@ -1,18 +1,29 @@
 use crate::tokenizer::RawKeyword;
 use crate::tokens::Punct;
+use std::fmt::Debug;
 use std::rc::Rc;
 
 /// A 2 element buffer of
 /// MetaTokens, this will use a
 /// "ring buffer"-esque scheme
 /// for automatically overwriting
 /// any element after 2
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct LookBehind {
     list: [Option<MetaToken>; 3],
     pointer: u8,
 }
 
+impl Debug for LookBehind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_list()
+            .entry(self.one())
+            .entry(self.two())
+            .entry(self.three())
+            .finish()
+    }
+}
+
 impl LookBehind {
     #[inline]
     pub const fn new() -> Self {

src/manual_scanner.rs

@@ -434,7 +434,7 @@ impl<'b> ManualScanner<'b> {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 /// All of the important state
 /// for the scanner, used to
 /// cache and reset a `Scanner`