From f85e5aac64784deac75950a1307f2ca802ad6765 Mon Sep 17 00:00:00 2001
From: Laurenz <laurmaedje@gmail.com>
Date: Thu, 8 Jul 2021 19:37:38 +0200
Subject: [PATCH] URL special case for line comments

---
 src/parse/scanner.rs                |   5 -----
 src/parse/tokens.rs                 |  13 ++++++++-----
 tests/ref/code/comment.png          | Bin 708 -> 1904 bytes
 tests/typ/code/comment.typ          |   5 +++++
 tools/support/typst.tmLanguage.json |  18 ++++++++++++------
 5 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/src/parse/scanner.rs b/src/parse/scanner.rs
index 1a0e3045b..fad44e89f 100644
--- a/src/parse/scanner.rs
+++ b/src/parse/scanner.rs
@@ -96,11 +96,6 @@ impl<'s> Scanner<'s> {
         self.peek().map(f).unwrap_or(false)
     }
 
-    /// Checks whether the remaining source starts with the given string.
-    pub fn starts_with(&self, string: &str) -> bool {
-        self.rest().starts_with(string)
-    }
-
     /// The previous index in the source string.
     pub fn last_index(&self) -> usize {
         self.eaten()
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index c9b7dc21e..f21967402 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -72,9 +72,9 @@ impl<'s> Iterator for Tokens<'s> {
             // Whitespace.
             c if c.is_whitespace() => self.whitespace(c),
 
-            // Comments.
-            '/' if self.s.eat_if('/') => self.line_comment(),
+            // Comments with special case for URLs.
             '/' if self.s.eat_if('*') => self.block_comment(),
+            '/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(),
             '*' if self.s.eat_if('/') => Token::Invalid(self.s.eaten_from(start)),
 
             // Other things.
@@ -216,7 +216,7 @@ impl<'s> Tokens<'s> {
                     self.s.eat_assert(c);
                     Token::Text(&self.s.eaten_from(start))
                 }
-                'u' if self.s.starts_with("u{") => {
+                'u' if self.s.rest().starts_with("u{") => {
                     self.s.eat_assert('u');
                     self.s.eat_assert('{');
                     Token::UnicodeEscape(UnicodeEscapeToken {
@@ -367,7 +367,7 @@ impl<'s> Tokens<'s> {
 
         // Read the fractional part if not already done.
         // Make sure not to confuse a range for the decimal separator.
-        if c != '.' && !self.s.starts_with("..") && self.s.eat_if('.') {
+        if c != '.' && !self.s.rest().starts_with("..") && self.s.eat_if('.') {
             self.s.eat_while(|c| c.is_ascii_digit());
         }
 
@@ -464,6 +464,10 @@ impl<'s> Tokens<'s> {
 
         Token::BlockComment(self.s.get(start .. end))
     }
+
+    fn maybe_in_url(&self) -> bool {
+        self.mode == TokenMode::Markup && self.s.eaten().ends_with(":/")
+    }
 }
 
 impl Debug for Tokens<'_> {
@@ -688,7 +692,6 @@ mod tests {
         t!(Markup[" /"]: r"\a"   => Text(r"\"), Text("a"));
         t!(Markup[" /"]: r"\u"   => Text(r"\"), Text("u"));
         t!(Markup[" /"]: r"\1"   => Text(r"\"), Text("1"));
-        t!(Markup[" /"]: r"\:"   => Text(r"\"), Text(":"));
         t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\""));
 
         // Test basic unicode escapes.
diff --git a/tests/ref/code/comment.png b/tests/ref/code/comment.png
index 6654e1255f466031825e7f32df41ac3d5d521bb2..bb8bf69ab794e054c4c65a67e5d96349c841b88c 100644
GIT binary patch
literal 1904
zcmZ`)dpOgJ8~={U+>Xg5a+#U)Yg7_L#Ee{1BzI1Thp0x5+r-ceCkfB_btto(vg%>W
z9BIpKm`f$)+FX_$5-Yh3ZF9?hI_Ix*`9073ywB%(pU?9?&wKq`KjZ1DB)?xC001R-
zH>a}z00QsgO~|eTA_}7w06^Bt-O0fhH?cUmm`t+O>{&^d9s?}fwcGdLq+38d%w>$F
zzV?99EhQAeWdI`KOMleiKKe9FaSwhzDugp)5et=r9ypcgg8RsCwo01rFaX0L4iHBu
z0ZP>W3(Qa<w592W=_!Hc(e~0D4lN-vIS?j)-om4dF1uC|+#H{4WuTaIxQtFtYv&4(
z5KIknvl8A=OaKlcI{oltPz@4H26W7cL*1OZoXcNjUFzgQ9i>Nw9YIG?DS};Ip~65I
zNA|+OE}myG0b|T%U`ekw>z>N_j^cY#T>=>eSAt+E;0rTV`Z^5yRz2Ewzf`6~kjN96
zOC^aXU3W~R27b}1dUHb)E91$b<YU$c@90WX_&Gu13gEqOFdjYHjT_z5b^};@->O{F
zBF;>f3%}bce$uw)O-QNv_p<+pviBh<k67`f&QDr%wV#{r!8T`nGLXdTKp-L#bCJ>M
zc`1^=g<cS75R;JX9hzVRs?#2x@9Hph+a9GD071>^fu~m&XkF?A=BfiUkBjKn0wEV_
zmAk*1NR2RIC<L%rrBiMN$0VeO4ym(qeDili?<Na@TO!Q^v32h%xMY=Vq4yhXo@FIj
z=CknF^cmY1cZ^F`wF-%5A5^WcCG{d+E*AkMSC6~4m#LRrq-Ghi6DN6%9omwD!bolk
z_PA@{M8{1mm$Af?Yq0;<`Tkj`E8>q$xR30WICn<>zg0l8!^%r6*Ry0m5gRNjyMN>4
z)=V=mjAp+T@x=-&_qmyOlP)^2L=I5e%#xqRFGs&<^KDC|vQoi6Z4ibYOgF>pLhV{g
ze~`<L+c1GZ8An7Tp=gSFM4TkF_Te2PargP=Jke~*cjeCxyXiRsxN(2ziO>?2p||Ac
zj(t=vv*E+^Y(25x@ru`jam9+hzLPfQPt0_&xFv0>^p&M^7W`C2kGmO>n%CAomYcKc
zd-CsV>L*2;Q>v4ELdJ~9=?Cq)avH1Y41?F8-(b&;<4>K!#y`zR71hogP!dh1M}j}u
ztKL)U>M<ZbTtP!=D`?Y@8Ms=Mp=OuHCLW#pm7z&7W5?$C9-`Fb{HJ}Wq-Ot6%xObE
zOGC#5pu$4MpE{gZQUhFy3+n+nw*A}-R2v~;8!XG9wadOjEi8d46T^(hRYpZ|!}n<i
zUFH&*T=a(>l{q~E`BAK~E{-%Q@2k*IdL!CWxiAf6I#K@%nlp?dixfuiEL(icOD1B?
zlJ7Cb+>2(OM~USf>q`S~dC`T%wX3}J%Itf6VVv&?%*EjCmhdc7S>aS`KZ0|f9vK^d
z*!Wc&$zwePvPC^J4xbbi)6`ApM4$*(0Hi-^>o_F9u8Xz0T^S=$A|AEJ?59c6Bp=P>
zbX*fL<>e1HEZCvCwKZv^_~v?g_2V65$KMb-fd38LuU1pD^;w0^Qk=!2AeGX)g9bV^
zt&|XA2G=2A@t_W4PQOE;Foa4%_JZpy%)_ojUxQj@Y#XB5J30Nj8Zpv}U(pAKlO>SH
zx?yb8-d?Br?-npMpVh?o%RInq(3-l@?eJM2oVClIL<;*M^wFJe&-MvJ0VA|bU=3l#
zQPU*uNLpa{q#n)4lOHWpdeMfN;QDMHc5S45{KXFwr!H{12j@HXS;ptS@e4UpQ6rjK
z;i&NwVT`v$QwN_!<PPc(d;46%tD8Qd3<K#WV|Z8TP2wfrl@J+my3$L;vliQTUy9RP
z1*27sP<VF$!zSyi@>X2#l??-B-?ocm+po+rZJLTVPINJgBHQMc`9~bq|GRrjsRd9k
z|4hL)WjaRa({zBS_G6{M8}2d`ur$F*kcGFXgO>1_bN0^JIOFB?le}V#+YGtiV;laq
zL%{*Ugf!mVHaR`J1>I(}87^Bku5B#*o(J_x-k06o8(V&xR`JltE*WbBoA;5*_#L){
z6}`98%SgyH>IEnIS@Ul{{xQxoM|Vz>th;w`FDrkXvs_>*MSaXnCk->0<Kul1a<*T@
zZ+~JvJ?%!<P1O(fs5=3;y*g*13d;~g9VM~SI@j$X(o<C1ZebPt3(#S!ayB{Mss$HM
z8YsOE#<rHJWX}M2snPbcS^Ye?-r|(F@{oV3-7#vj3JqBqR)jE3nRXSPOPAHZy4<;=
z+7bft=YtYE8nNKkCTpod{4j6&X~xTHzi%?LBm>G^h@5qG5yX)v@yq81cf0eqSG=wY
zcJ@~LW*f<<%NqZXroYkusa?B-ECq)@Q+Q|oECJwNeu=5CE$qHYsk^hMQ>|k-;eV%?
BaUlQz

literal 708
zcmeAS@N?(olHy`uVBq!ia0y~yU}OQZ7jOUxhVwIhXE87^b$hxvhE&{od+TEEVh0ho
zhqoC^6092d+W0)!AM+KkFJZpK_^zSE;5MUbz@aH0-Z$F{S!l1-UAsxC_}t`^((yIP
zk9MZHfPnw9Gfa#u0uDesP>Gg&Cl3DWuh-uZShC7{y~^2hYvg_f8>hD#%s%>d>vjd#
z^y6U_f(NT-Z@b>i>Upq6k7Lp!PuK72EYrE0maLq9|LkGj#7&0cES_gqRhsiRpS`rz
z=9|f$whdP<)yc7G*W~)1V08?#_k6YOv%w9WwR7hdI|k^?Enau;+~K$RO@6=6_ss7x
zxXYGk&0pSf@K=n$tlR6Wt@(E<YTU`M^LhWbMPcjle?2u~jzQ6CJz9S~_MV+|F6_9D
zfL-+F?XF$dkFDA3<{5S0d$r#5RWJB+Kj<`>_$juyb}6<P@CQyhX4qoT)@-l(VJR1f
zLIVR67UGb(@{#^^#ihmjypnHkTl>D|`?rm0tsmliCT+Q<rg?39^75VAPH*g&jdSeq
z-FPQnR8jVz=cMgw;w*cY&Yt#%&v7g7Oog27n-`UF96aSC@aOT#Vy6iqHx~9kn&&!U
z)p2QGj!7|k?w{G3LTuitz7;T%Y=2i+Zm#H|rQ5H{>&w6UE~n$AIj8eBKe<#k&E)-j
z;aN(PeML2w@w(|gog`+ddbGG|(w~<clSHg{ubJdjIY~r&wcX-r*-!cMK0K<QnEgKH
z^_?AFlRrDZpHqDJBIge~^r(f!^pEQt;51O*eLM{{@?k14QUzS%VZEY&prGKz>2=JZ
WYs>yOPK+-GB~MRRKbLh*2~7Y^dll~h

diff --git a/tests/typ/code/comment.typ b/tests/typ/code/comment.typ
index 25180211f..694600d6c 100644
--- a/tests/typ/code/comment.typ
+++ b/tests/typ/code/comment.typ
@@ -14,6 +14,11 @@ C/*
 #test(type(/*1*/ 1) //
 , "integer")
 
+---
+// Line comments have a special case for URLs.
+https://example.com \
+https:/* block comments don't ... */
+
 ---
 // End should not appear without start.
 // Error: 1:7-1:9 unexpected end of block comment
diff --git a/tools/support/typst.tmLanguage.json b/tools/support/typst.tmLanguage.json
index aaad9ba8f..e1f416535 100644
--- a/tools/support/typst.tmLanguage.json
+++ b/tools/support/typst.tmLanguage.json
@@ -6,12 +6,6 @@
   "repository": {
     "common": {
       "patterns": [
-        {
-          "name": "comment.line.double-slash.typst",
-          "begin": "//",
-          "end": "\n",
-          "beginCaptures": { "0": { "name": "punctuation.definition.comment.typst" } }
-        },
         {
           "name": "comment.block.typst",
           "begin": "/\\*",
@@ -38,6 +32,12 @@
     "markup": {
       "patterns": [
         { "include": "#common" },
+        {
+          "name": "comment.line.double-slash.typst",
+          "begin": "(?<!:)//",
+          "end": "\n",
+          "beginCaptures": { "0": { "name": "punctuation.definition.comment.typst" } }
+        },
         {
           "name": "constant.character.escape.content.typst",
           "match": "\\\\([\\\\/\\[\\]{}#*_=~`$-.]|u\\{[0-9a-zA-Z]*\\}?)"
@@ -191,6 +191,12 @@
     "code": {
       "patterns": [
         { "include": "#common" },
+        {
+          "name": "comment.line.double-slash.typst",
+          "begin": "//",
+          "end": "\n",
+          "beginCaptures": { "0": { "name": "punctuation.definition.comment.typst" } }
+        },
         {
           "name": "punctuation.separator.colon.typst",
           "match": ":"