diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index ae9cfdb1b..6a8b8103a 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -1001,18 +1001,20 @@ mod tests { #[test] fn test_parse_simple_nodes() { - t!("" => ); - t!("hi" => T("hi")); - t!("*hi" => B, T("hi")); - t!("hi_" => T("hi"), I); - t!("hi you" => T("hi"), S, T("you")); - t!("\\u{1f303}" => T("🌃")); - t!("\n\n\nhello" => P, T("hello")); - t!(r"a\ b" => T("a"), L, S, T("b")); - t!("`py`" => R!["py"]); - t!("`hi\nyou" => R!["hi", "you"]); - e!("`hi\nyou" => s(1,3, 1,3, "expected backtick")); - t!("`hi\\`du`" => R!["hi`du"]); + t!("" => ); + t!("hi" => T("hi")); + t!("*hi" => B, T("hi")); + t!("hi_" => T("hi"), I); + t!("hi you" => T("hi"), S, T("you")); + t!("special~name" => T("special"), T("\u{00A0}"), T("name")); + t!("special\\~name" => T("special"), T("~"), T("name")); + t!("\\u{1f303}" => T("🌃")); + t!("\n\n\nhello" => P, T("hello")); + t!(r"a\ b" => T("a"), L, S, T("b")); + t!("`py`" => R!["py"]); + t!("`hi\nyou" => R!["hi", "you"]); + e!("`hi\nyou" => s(1,3, 1,3, "expected backtick")); + t!("`hi\\`du`" => R!["hi`du"]); t!("```java System.out.print```" => C![Some("java"), "System.out.print"]); t!("``` console.log(\n\"alert\"\n)" => C![None, "console.log(", "\"alert\"", ")"]); diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index d566363cc..e333968b4 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -265,6 +265,9 @@ impl<'s> Iterator for Tokens<'s> { '_' if self.mode == Body => Underscore, '`' if self.mode == Body => self.read_raw_or_code(), + // Non-breaking spaces. + '~' if self.mode == Body => Text("\u{00A0}"), + // An escaped thing. '\\' if self.mode == Body => self.read_escaped(), @@ -279,7 +282,7 @@ impl<'s> Iterator for Tokens<'s> { let val = match n { c if c.is_whitespace() => true, '[' | ']' | '{' | '}' | '/' | '*' => true, - '\\' | '_' | '`' if body => true, + '\\' | '_' | '`' | '~' if body => true, ':' | '=' | ',' | '"' | '(' | ')' if !body => true, '+' | '-' if !body && !last_was_e => true, _ => false, @@ -439,7 +442,7 @@ impl<'s> Tokens<'s> { fn read_escaped(&mut self) -> Token<'s> { fn is_escapable(c: char) -> bool { match c { - '[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' => true, + '[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' | '~' => true, _ => false, } } @@ -646,6 +649,7 @@ mod tests { t!(Body, " \n\t \n " => S(2)); t!(Body, "\n\r" => S(2)); t!(Body, " \r\r\n \x0D" => S(3)); + t!(Body, "a~b" => T("a"), T("\u{00A0}"), T("b")); } #[test]