From 67047047e82c564d7701c3505c85db6e34223763 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sat, 30 Jan 2021 15:24:11 +0100 Subject: [PATCH] =?UTF-8?q?Interpret=20two=20backticks=20as=20single-backt?= =?UTF-8?q?ick=20block=20=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parse/tokens.rs | 23 ++++++++++++------ src/syntax/mod.rs | 5 ++-- src/syntax/node.rs | 53 +++++++++++++++++++---------------------- tests/lang/ref/raw.png | Bin 4071 -> 4459 bytes tests/lang/typ/raw.typ | 15 ++++++++---- 5 files changed, 54 insertions(+), 42 deletions(-) diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 71e71e3e8..405352c3a 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -227,6 +227,11 @@ impl<'s> Tokens<'s> { backticks += 1; } + // Special case for empty inline block. + if backticks == 2 { + return Token::Raw(TokenRaw { text: "", backticks: 1, terminated: true }); + } + let start = self.s.index(); let mut found = 0; @@ -723,21 +728,25 @@ mod tests { #[test] fn test_tokenize_raw_blocks() { + let empty = Raw("", 1, true); + // Test basic raw block. + t!(Markup: "``" => empty); t!(Markup: "`raw`" => Raw("raw", 1, true)); t!(Markup[""]: "`]" => Raw("]", 1, false)); // Test special symbols in raw block. - t!(Markup: "`[func]`" => Raw("[func]", 1, true)); - t!(Markup[""]: r"`\`` " => Raw(r"\", 1, true), Raw(" ", 1, false)); - - // Test more backticks. - t!(Markup: "````🚀````" => Raw("🚀", 4, true)); - t!(Markup[""]: "````👩‍🚀``noend" => Raw("👩‍🚀``noend", 4, false)); - t!(Markup[""]: "````raw``````new" => Raw("raw", 4, true), Raw("new", 2, false)); + t!(Markup: "`[brackets]`" => Raw("[brackets]", 1, true)); + t!(Markup[""]: r"`\`` " => Raw(r"\", 1, true), Raw(" ", 1, false)); // Test separated closing backticks. t!(Markup: "```not `y`e`t```" => Raw("not `y`e`t", 3, true)); + + // Test more backticks. + t!(Markup: "``nope``" => empty, Text("nope"), empty); + t!(Markup: "````🚀````" => Raw("🚀", 4, true)); + t!(Markup[""]: "`````👩‍🚀````noend" => Raw("👩‍🚀````noend", 5, false)); + t!(Markup[""]: "````raw``````" => Raw("raw", 4, true), empty); } #[test] diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 41fba1345..409e8cbf2 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -63,9 +63,10 @@ mod tests { roundtrip("= *Ok*"); // Raw. + roundtrip("``"); roundtrip("`lang 1`"); - test("`` hi``", "`hi`"); - test("`` ` ``", "```"); + test("``` hi```", "`hi`"); + test("``` ` ```", "```"); } #[test] diff --git a/src/syntax/node.rs b/src/syntax/node.rs index d45e59528..f7625036e 100644 --- a/src/syntax/node.rs +++ b/src/syntax/node.rs @@ -66,11 +66,11 @@ impl Pretty for NodeHeading { /// A raw block with optional syntax highlighting: `` `raw` ``. /// -/// Raw blocks start with an arbitrary number of backticks and end with the same -/// number of backticks. If you want to include a sequence of backticks in a raw -/// block, simply surround the block with more backticks. +/// Raw blocks start with 1 or 3+ backticks and end with the same number of +/// backticks. If you want to include a sequence of backticks in a raw block, +/// simply surround the block with more backticks. /// -/// When using at least two backticks, an optional language tag may follow +/// When using at least three backticks, an optional language tag may follow /// directly after the backticks. This tag defines which language to /// syntax-highlight the text in. Apart from the language tag and some /// whitespace trimming discussed below, everything inside a raw block is @@ -82,22 +82,21 @@ impl Pretty for NodeHeading { /// `raw` /// ``` /// - An optional language tag may follow directly at the start when the block -/// is surrounded by at least two backticks. -/// ```typst -/// ``rust println!("hello!")``; -/// ``` -/// - Blocks can span multiple lines. Two backticks suffice to be able to -/// specify the language tag, but three are fine, too. -/// ```typst -/// ``rust +/// is surrounded by at least three backticks. +/// ````typst +/// ```rust println!("hello!")```; +/// ```` +/// - Blocks can span multiple lines. +/// ````typst +/// ```rust /// loop { /// find_yak().shave(); /// } -/// `` /// ``` -/// - Start with a space to omit the language tag (the space will be trimmed -/// from the output) and use more backticks to allow backticks in the raw -/// text. +/// ```` +/// - Start with a space to omit the language tag (the space will be trimmed +/// from the output) and use more backticks to allow backticks in the raw +/// text. /// `````typst /// ```` This contains ```backticks``` and has no leading & trailing spaces. ```` /// ````` @@ -107,24 +106,22 @@ impl Pretty for NodeHeading { /// given, a few things would become problematic or even impossible: /// - Typical multiline code blocks (like in the example above) would have an /// additional newline before and after the code. -/// - Raw text wrapped in more than one backtick could not exist without -/// leading whitespace since the first word would be interpreted as a -/// language tag. +/// - The first word of text wrapped in more than three backticks would always +/// be interpreted as a language tag which means that text without leading +/// space would be impossible. /// - A single backtick without surrounding spaces could not exist as raw text /// since it would be interpreted as belonging to the opening or closing /// backticks. /// -/// To fix these problems, we trim text in multi-backtick blocks as follows: -/// - We trim a single space or a sequence of whitespace followed by a newline -/// at the start. -/// - We trim a single space or a newline followed by a sequence of whitespace -/// at the end. +/// To fix these problems, we trim blocks with 3+ backticks as follows: +/// - A single space or a sequence of whitespace followed by a newline at the start. +/// - A single space or a newline followed by a sequence of whitespace at the end. /// /// With these rules, a single raw backtick can be produced by the sequence -/// ``` `` ` `` ```, ``` `` unhighlighted text `` ``` has no surrounding -/// spaces and multiline code blocks don't have extra empty lines. Note that -/// you can always force leading or trailing whitespace simply by adding more -/// spaces. +/// ```` ``` ` ``` ````, ```` ``` unhighlighted text ``` ```` has no +/// surrounding spaces and multiline code blocks don't have extra empty lines. +/// Note that you can always force leading or trailing whitespace simply by +/// adding more spaces. #[derive(Debug, Clone, PartialEq)] pub struct NodeRaw { /// An optional identifier specifying the language to syntax-highlight in. diff --git a/tests/lang/ref/raw.png b/tests/lang/ref/raw.png index 8b68922a515c5bb03bfff681aa5e3de1cae2760a..2a5b083db0a1b73e6c6d726653d31a60d1e08989 100644 GIT binary patch literal 4459 zcmai&c{J4R`^SeyJ|Sl8WUQrZSt3i8vF~CeGIkHe7(0)pVMddLqOv8-*tf^h!jxrf zp(rFvOqQN7_Qn>P!O!!@?{}W>`9071o^##jT<5;e{m*^B&-;2^*L_crb;gUrQ4{jivjvu3SCCwnPI>g+KFKsZmW4o+m_3p-PUe+Cl-zzn z>q-^}68>9slEkgpL4mntq-{fJR?$f~s=8W*VTGUOXM!W)zw7Z7avbD97vrTn5#N?! z66DN7Q^{=|a~8N<=}7;}kyMgI#^wKUKL(ZNU!W@^;=ed93NxLC|h71$l)a zw)m2@UJS+-DVLT&`@{G@UhnX7r!Jd+Jlw^`_jL?vwcIba;55_i-{LvOUhYx~2D?|O zmiRs43Boh6xtt~z&+XN!k!pi};;MU*X<^qTRt{5ZGSWJ`i^9 zO2Xhr)VZowH`i8(#{&>l)N)H%Jg-35(7H@Eg~luKgspKo#L z!RiY>#pBzXYL_Ei34ER$so#=_X`~M^a)ZgibEsZkWsbw!`XvH_)z z>ZZN?jtaji*kB^o@xd&*?vIAuWNa?UP7K4U8fH>&&FqbnwbfjV4Pf87bzjzQ7e0Dt z8B1=8(aPZjWOa@5XGgPY!(por>o0Isj7PXqgiu5mK(@P&^g5BK?Xl8oBeSN4FL66Q zvOKx~RG6S@NGHa|e8MCZ!;JVaOuYTt(wV;=DJ1n)!Sp|lLl{3}V4^HIe{J`9uWyC# zB&?>iAHvlm*<&gNikz{}C!o0-%S*(s#V)W&F!E&5{xYc1CN*NxBvAnk1C{sV*Z7DO zqYG46$JqT5U%UuQuak*5h`xWKu3txLD7%MOSqkoB-lSRvs*uH22-7T1R90F~Q0b{7 z8D;Mb*>J`UzF;jMtJZU@KPYtVl) z3Vi$nc+^vBwyB@~!Fgb*rlv=b-q~_+4X=@bu`&9R*>%ZUnsEc1<6zUmzA=Ae!MtTB z9KeLcoEUy4(?hKi2&@v!znCxAJYNY8G{!D=y>u0s)BS(Gw0=s$nh)Mf2#(7yI5@umSF2CI7&A8O<--clNCWEO-L9Rxu*+BFOWKNFW70m|2N(s03O zv;*ntJ`aeR7#dk>F6c<=$l#=%cUUj(yQbqRz8DJZCPESoyCyHo8lq?s^Nb4S_pxfR zKX{^a7P}Q=m;XeM^au3E*>O5VyAEJZrR3TVokUQg;w!Ec)l4dCis+_b9e&cHRSXJ* z{+RSVxfPAhR{!u_Te!tSmEY&Na2>G`j$m>uK}944jp#_cC2JhSaOuzU{1TCXK^qUo zggWFN3zzxjT3jtpi(rP7H-+rtLGPC~Te#Mxz`6#uMx#hn&4WcL=9Z^kj9iOvjn#%C zRED=`$%rU)VNf!P#dsQp6oU@AtGUdkt@9+~RC!tP0|AB|y7!c_gCGJ)%XhbnMAv*z zFRX=eokYxG$CwxnRB>b9F zimoA?=ztMIa83**{NdGt9Z?u=d(3&PLG80@w1;G?bsgO$N;K)Bm2N9JAtk6z#tM0u zjz0I+FS)gc6tv67$GCxSeKUa@Gj5@!Y1&oq8Mgnbm@Nc@Igo3cV)&3m}VHQC}bL#duBjwXY5NJ^rFfo z^1bwB?#sDV4v3#i;!;d51ewrxTQIS~aUohu;|JIra=o!5@G~N7_ht$Bg+lMSI4Pjn% ziC?k(?Ps}c{}l>oYQRR&gU;6?8*0{Rc%>Qs*xPwNu%uY_@5YSv-^F6oBi~(ASIeSE z3CaITN~dC@UTHCT6;OX88K7u?n#hJ_F26^H;YxVi1?yA|4rQ z3Z}pVIA$}Kq#msli)=k2_A7tNq_VfN(R5n5+wKrafzr)sev4XjN@#p9K^ zp7gQr-Y;5jwTa^DSjK3G_>%=!3y<1t-Zke7Ocv>4t^)l8Nl@!R^VUsZ7VXg6YgEyT zfsaRCr#c#r!`$1_c4N4hhNHjs7K`VN1 zIWeAS`=3e>XUKHE=G?d8LB1q<;ezT)Y%NdA#sK0>S^RnsB zx?pnpb9e}?%+b3MFB77&z(=bCS#2;&a4wwF&nsh`Je#R5A;wl&`}x=+5Q<@iAoquH z7k!h)s-`E|!Jp;hY9-%#xwu3mR4~t`q)2@maY70ujZNl7yiHz95dFF6y zU6?<+JQ5wrNYxzd%To7Zw_Txc@x&Y6;+Ch2^?lQ79IW0ds=p`1!jDaTBd?F z_KLJ@i)gRYeAl_TXlTN(6FK}ft^;<_Q{0dEKoR#cRu z@Yr;`$+n#>uH2bDa?gU!lC%pn#lp$zEY;f1OeMZ?@_28}Adi&!NXIWOhVSONJ)57h z`Lx*MPwAl*{06?4%e1RnQ*vmFn|;Up{3=L&9WCP=G`J$<85Fyf4Y)lYEQ}1Ocq0IY z=L=W&gKp6g;Zw}cE9inNGh|h72k9Yahd1U$Uy-Y^&1-%p*z_%ZF-R!S2x3|;fhC-E zv1yQRZ`xvQl%q0dE0Qviev}tBWJ;@2>ls-No?LA!*FFD)^;(lXn835OEF1qeHCXvx zUaM0k`>aDy&FBwtm+JHDJf3#r$1Uvz#*poFW3SWU6J!@3HibeZ2Umh4?Up zP)h?vm8tTz_RH@DtI=(~t8$-10{v;mO^$fvimX%LjK3Mj{7ka>rHEApiI$V2wY+6xFXcXC~)#7{Xx@cccxOo56=panBcAxg!{65BcavpE+>1 z@7u{doaa_@vmDHwAsGAIeQrH-x4krpw6SFHifV7kWek`A76i0ua$YZ~=!4$A(PePSv{>09aqsV^uTg#xb+ z!+(!@8Cxl+2Nz^5T?Wb5ApBR7^khn{CZUPa`Be0ESuSdhMd`ASDNuUH<65Crlqxt} z`N=QtdQQgJ=~1I$L3O-miphIZgLQA*a~e|Mu48>hpqNY6!rm=_lAYfR3TPX6g94*p zc0h4D-?!qlUe}x{L$zu6{FpKNqIF}&ru5==v|?VP;lofp$vN;C*3kydG;Ln}vs+I# zA#^Cx^Ta!V=)jXaF9ohk9$F)@eA@AkvI5t6t%r>&QD0-ip9CY*9R?_Q(9ogO{X|hmDIfGNZUiHF{>xNan6cwGA*C6Ed3gP4h}ny*DW@4*w5hv^lqhAd{Z1 zn$Ta)R4xm?ii2+=1FB{o%P@llerBZ1X4m}O3LJbXn1wO#OA&>Adz$j!<`DnNB>#J{ jvsLo{h8t#@{5xU_>^g`oVK$k+rYPpdHb%7u9tr;g7vLG; literal 4071 zcmbtXX*d*)x1KRm24M!tHW_=C#+rQ{+fbH>l6nLU5~Pb#C|5cN$0pE~8_Kt<@bBtYdPv>! zcSH#;u6X=UD-po3^8WVFc_3A9El+P``bXfaP=NFv4;_?;Ba&zT8k6Jow01l(8su$pLvmpIFi zH(ZQswZIsZp`mF{-y_qWrO55ov^V~!9j^lgBXYNGk1Jqnwwrswy8GZ0;hnixGbc8h ztQ|OU+C1~?k%4`{(6Ki-c*n{@-fVT96wlvbCGpR6?f!+mB_E9&cU1npd7^X>y;~|8 z5O{i8Ptc8)sl)U_%hQs$X{<4243_UQ*%&;9SyN>cja%X$V^a7ejmpJHR53{& z@|fY4_9U5*Wvb1vR`nQ? z%zClx@t3Us7agWb{7(n<+vPA1YUDL=^&~g|bPXxiAgVuMD1(yW<%=-^1?pvs4{T?# zvJHsyy#>#lPs|23mz9=qq`2%5@3#A@VXgOXIrDY(8?HKF9psW7Z9zbE--;~v?P8_wr+XConzXl82};me$86R6x3W~e4=l!GEvJj(n`e@q|kMA zXtHLYM0-9-RN-7L23OZt%xP+Zw$QN%r=Ak4%zbF5qcfy0!-}!CC!#^vmj%m`fspFg z6{DVD;@se1b0Fi}yS&Qca_$rjo}>xZ50Jl^)cxx}&cY{s(7lAXCMq#Lvil@!zRs$L ztowDiD^@V8zV0~OpW|{(-R0H=b=HmtHZnSyPg)@&e{SL`xRo{0HDEu%G_tk~lKT9A z;I;zxcD#>QC8G`WVU;-p^t*+9opQ_puF=cg@xY#k1JQj|c1KT?_2V89g{j<(1)DEb z5dI70G1!XRM%dfP;7rD-k>{C6?!O*0oRss`mgH;#TeP%YyX+}7Pg(elV=s!`mNq5> zus3JJ)X08vqYGhj*sU>p27Rv>){c{$9_|%vWP1=yQ+yAn;^9hi?%7my_P(e?V&jIw zx2!KBit8r1NW&-&Qxx&N@IoFe)Nx;65+Sxr>$q*K5Sd63Rc(`|lcM1Q{s$L z$IZt?uf%R~MbQ-aIt|}MK=x|m&oSApWh&!X8sz!Jgt(@VcJys4+D~Y*>h1J(V+T*x z!>A;OLc$0kvbI11lCCnI4n5Pp2pB*N>^28nmIMTe^YOtcb85TJo$3g_%ACJ-1UiDv z{*8hS9mrJCGkuXmdoX0PH{%iIkOPaG27DSnYD2uU>pV(`n!J=8R+~{QjDjqLdgm_( zJyO(!_0j#241ZoStj?s>2M}n7iSLr+G~OMhu5UL26~ZR zU!Q*a_voc-_Fh1jf~#p4y|4IL&^Wow@Hcn|nLHGchJJ`%yEqN=yStDi*$1(^p%w_6 zxtH;dixMIngC3`O&CIktPL+LueVS7Oc33x<8UiN1;TlXow~5Jbtw5zb6#fv5L z$9|g~oG!OP)PQ{X<=*&F=kcKj-7H%<*nXvwtu+`oJ zv*4u|z;>UjRMaTBb1#B5dDdkP*uN(nU zO}8J{&X;!SEBGsC(@)<$lPu2mE<`F0seO{6FrV@$O2nYNM1x=V)Q5a+{^#GDVDbru z4uY#|lD^IYm3_YqV5e(19^5>TJ(OkAzZv~7GkH?NKxQFOAD4YlZS}Er_3ZrSC2U~F z*G&3DA<&EX)>?}sZ+)(=d$51EgM;GKhi_&o4(lYN8 zE!g%-p!0>??v0`49eO;drDIu?r9KXa^m68zsfNbNt_rYw_`NE|^oI`+K#O-oi_QJy zSt50x%}IBJ-b`?7Fx`;ja-?E#VFV`2zo6g`3_oD&GBr7h*8#~)VJ&rG@ z5i;fCk{lW*3y?lR*XeWE;mC<^Se71-9S(aLo1;QRDKQ@y_)OF$8)oZN@9V+>e~;Qo>E}EjfNCUs6K8scrPI z^B~~Jy$gTK8gT4rXmQA$R5u)yeR>^+XnY}~gN=3=P<^R49)lAb~x%om+IdnHVHf1 z6Ox!rjqxj=4@#;)!LPE``Bq{(;;cGbgQHTftlpfRBOF)7aa=!*r~^&pw+aALoC1eu z_?vgePcEKol*okbdqKK|<3^+K*RFz%cZ0BM6x9}c-@{AiYZFw!jC#Sgu=4=&<9-eF zO8duftXl>%I}IW|a(}z}>C~Ysgft=C9~+S?%%e}$OwXO5jN$b)&WE+C)lDm|kmFfrl3T>{BjnACX(K5$Z;$L zOxBJkyCnFTU~*>+e6Xt;s>~#z2lqtLTF85HnChbH;d^y=p~HEg2aPz$aB8<57Yzg| zrHge6aSFJ~+BMInGDGIB!h{rRG13f;0{_aE2JHew)S-e%Leb8oO>vmgy0LEH_i?HE z_)sTa&eTbu6vpqn*qCLR&$@lc!H*v?dOu;QBdtHbL*`vzL2Q&XOpFS)I+{&n#2j=$ zPQDKP&4MlogC7KdJ(H|Q>!FKQQBE`C_lTd534mL7u3^j9JSQ)LQGegXgL0d85Y&T8 zBTSI4OcG~MOD?^4?Mdvh+6Z~hx7WY(r0GiQsTk*pIZe%+BONJ%xvBB4?~PNf`xOFn z)^Ya6TQDf(s4dp^WW8;38iys<4eSP=ovD|- zQzGY@6;puo$YpP|(d6Ne4;Zu(xjix@8evCuk{ExKHKSU?$#f)uT1yR~InLA^B*xi) z@Rie8R2mWbqS2<5gI_Vupk4WF@O%W*uPFA;?wfS*wC|}IK}ZGJuM=qdBljf;67Kd! zlK`oNuT87=@PhNhxIBg-y;0&07X$~N_LPdaz5$S~5RK6H?>TV1`%0zETfV7>|EMqi z>clOQeAW%l?vqTv^)xfdwc7)LaHfU$XO7u>-_v30jD7K;$GhG7WXf^SM-;(3-Kb*d z734TY{mx65g*2*z<*hzN|7}KkvbOnL19$^Hmurt1t2mu@rZV5f@N)#4b81#qJhpN9 zJi8nFU4hPz`tTo+85F8CbF8yB`c5!4!en!}0{y=#^R6_wydmS{u+0R@Nm0qnPtavc zWG;bUf$1hg`(70?_20kjRLljnw2{}sR_5GwOCNP8*3Rm!`(~x56SY-==nPE>k>f_w z4qgc$a$0tdbUGSu4A`hcl(^}TGZx}!Oxdd$BG^k|p33*qF!(zk8E{k`RKq zQO;l@si55NuT@oQ)MNaq&GR2}{vXn7Jmw$<15wb3%lt}?nK72Hy*6wGb^ev$&5SLL J2q@R2{{VWShSdN7 diff --git a/tests/lang/typ/raw.typ b/tests/lang/typ/raw.typ index a1f796b5c..36149dc89 100644 --- a/tests/lang/typ/raw.typ +++ b/tests/lang/typ/raw.typ @@ -1,22 +1,27 @@ #[font 8pt] +// No extra space. +`A``B` + // Typst syntax inside. -`#let x = 1``[f 1]` +`#let x = 1` \ +`#[f 1]` // Space between "rust" and "let" is trimmed. -The keyword ``rust let``. +The keyword ```rust let```. // Trimming depends on number backticks. +<``> \ <` untrimmed `> \ -<`` trimmed ``> +<``` trimmed ```> // Multiline trimming. -``py +```py import this def say_hi(): print("Hi!") -`` +``` // Lots of backticks inside. ````