cargo/util/
frontmatter.rs

1type Span = std::ops::Range<usize>;
2
3#[derive(Debug)]
4pub struct ScriptSource<'s> {
5    /// The full file
6    raw: &'s str,
7    /// The `#!/usr/bin/env cargo` line, if present
8    shebang: Option<Span>,
9    /// The code fence opener (`---`)
10    open: Option<Span>,
11    /// Trailing text after `ScriptSource::open` that identifies the meaning of
12    /// `ScriptSource::frontmatter`
13    info: Option<Span>,
14    /// The lines between `ScriptSource::open` and `ScriptSource::close`
15    frontmatter: Option<Span>,
16    /// The code fence closer (`---`)
17    close: Option<Span>,
18    /// All content after the frontmatter and shebang
19    content: Span,
20}
21
22impl<'s> ScriptSource<'s> {
23    pub fn parse(raw: &'s str) -> Result<Self, FrontmatterError> {
24        use winnow::stream::FindSlice as _;
25        use winnow::stream::Location as _;
26        use winnow::stream::Offset as _;
27        use winnow::stream::Stream as _;
28
29        let content_end = raw.len();
30        let mut source = Self {
31            raw,
32            shebang: None,
33            open: None,
34            info: None,
35            frontmatter: None,
36            close: None,
37            content: 0..content_end,
38        };
39
40        let mut input = winnow::stream::LocatingSlice::new(raw);
41
42        if let Some(shebang_end) = strip_shebang(input.as_ref()) {
43            let shebang_start = input.current_token_start();
44            let _ = input.next_slice(shebang_end);
45            let shebang_end = input.current_token_start();
46            source.shebang = Some(shebang_start..shebang_end);
47            source.content = shebang_end..content_end;
48        }
49
50        // Whitespace may precede a frontmatter but must end with a newline
51        if let Some(nl_end) = strip_ws_lines(input.as_ref()) {
52            let _ = input.next_slice(nl_end);
53        }
54
55        // Opens with a line that starts with 3 or more `-` followed by an optional identifier
56        const FENCE_CHAR: char = '-';
57        let fence_length = input
58            .as_ref()
59            .char_indices()
60            .find_map(|(i, c)| (c != FENCE_CHAR).then_some(i))
61            .unwrap_or_else(|| input.eof_offset());
62        let open_start = input.current_token_start();
63        let fence_pattern = input.next_slice(fence_length);
64        let open_end = input.current_token_start();
65        match fence_length {
66            0 => {
67                return Ok(source);
68            }
69            1 | 2 => {
70                // either not a frontmatter or invalid frontmatter opening
71                return Err(FrontmatterError::new(
72                    format!(
73                        "found {fence_length} `{FENCE_CHAR}` in rust frontmatter, expected at least 3"
74                    ),
75                    raw.len()..raw.len(),
76                ).push_visible_span(open_start..open_end));
77            }
78            _ if u8::try_from(fence_length).is_err() => {
79                return Err(FrontmatterError::new(
80                    format!(
81                        "too many `-` symbols: frontmatter openings may be delimited by up to 255 `-` symbols, but found {fence_length}"
82                    ),
83                    open_start..open_end,
84                ));
85            }
86            _ => {}
87        }
88        source.open = Some(open_start..open_end);
89        let Some(info_nl) = input.find_slice("\n") else {
90            return Err(FrontmatterError::new(
91                format!("unclosed frontmatter; expected `{fence_pattern}`"),
92                raw.len()..raw.len(),
93            )
94            .push_visible_span(open_start..open_end));
95        };
96        let info = input.next_slice(info_nl.start);
97        let info = info.strip_suffix('\r').unwrap_or(info); // already excludes `\n`
98        let info = info.trim_matches(is_horizontal_whitespace);
99        if !info.is_empty() {
100            let info_start = info.offset_from(&raw);
101            let info_end = info_start + info.len();
102            source.info = Some(info_start..info_end);
103        }
104
105        // Ends with a line that starts with a matching number of `-` only followed by whitespace
106        let nl_fence_pattern = format!("\n{fence_pattern}");
107        let Some(frontmatter_nl) = input.find_slice(nl_fence_pattern.as_str()) else {
108            for prefix_len in (2..=(nl_fence_pattern.len() - 1)).rev() {
109                let Some(frontmatter_nl) = input.find_slice(&nl_fence_pattern[0..prefix_len])
110                else {
111                    continue;
112                };
113                let nl_len = "\n".len();
114                let close_len = prefix_len - nl_len;
115
116                let _ = input.next_slice(frontmatter_nl.start + nl_len);
117                let close_start = input.current_token_start();
118                let _ = input.next_slice(close_len);
119                let close_end = input.current_token_start();
120                let fewer_dashes = fence_length - close_len;
121                return Err(FrontmatterError::new(
122                    format!(
123                        "closing code fence has {fewer_dashes} less `-` than the opening fence"
124                    ),
125                    close_start..close_end,
126                )
127                .push_visible_span(open_start..open_end));
128            }
129            return Err(FrontmatterError::new(
130                format!("unclosed frontmatter; expected `{fence_pattern}`"),
131                raw.len()..raw.len(),
132            )
133            .push_visible_span(open_start..open_end));
134        };
135        let frontmatter_start = input.current_token_start() + 1; // skip nl from infostring
136        let _ = input.next_slice(frontmatter_nl.start + 1);
137        let frontmatter_end = input.current_token_start();
138        source.frontmatter = Some(frontmatter_start..frontmatter_end);
139        let close_start = input.current_token_start();
140        let _ = input.next_slice(fence_length);
141        let close_end = input.current_token_start();
142        source.close = Some(close_start..close_end);
143
144        let nl = input.find_slice("\n");
145        let after_closing_fence = input.next_slice(
146            nl.map(|span| span.end)
147                .unwrap_or_else(|| input.eof_offset()),
148        );
149        let content_start = input.current_token_start();
150        let extra_dashes = after_closing_fence
151            .chars()
152            .take_while(|b| *b == FENCE_CHAR)
153            .count();
154        if 0 < extra_dashes {
155            let extra_start = close_end;
156            let extra_end = extra_start + extra_dashes;
157            return Err(FrontmatterError::new(
158                format!("closing code fence has {extra_dashes} more `-` than the opening fence"),
159                extra_start..extra_end,
160            )
161            .push_visible_span(open_start..open_end));
162        } else {
163            let after_closing_fence = strip_newline(after_closing_fence);
164            let after_closing_fence = after_closing_fence.trim_matches(is_horizontal_whitespace);
165            if !after_closing_fence.is_empty() {
166                // extra characters beyond the original fence pattern
167                let after_start = after_closing_fence.offset_from(&raw);
168                let after_end = after_start + after_closing_fence.len();
169                return Err(FrontmatterError::new(
170                    format!("unexpected characters after frontmatter close"),
171                    after_start..after_end,
172                )
173                .push_visible_span(open_start..open_end));
174            }
175        }
176
177        source.content = content_start..content_end;
178
179        if let Some(nl_end) = strip_ws_lines(input.as_ref()) {
180            let _ = input.next_slice(nl_end);
181        }
182        let fence_length = input
183            .as_ref()
184            .char_indices()
185            .find_map(|(i, c)| (c != FENCE_CHAR).then_some(i))
186            .unwrap_or_else(|| input.eof_offset());
187        if 0 < fence_length {
188            let fence_start = input.current_token_start();
189            let fence_end = fence_start + fence_length;
190            return Err(FrontmatterError::new(
191                format!("only one frontmatter is supported"),
192                fence_start..fence_end,
193            )
194            .push_visible_span(open_start..open_end)
195            .push_visible_span(close_start..close_end));
196        }
197
198        Ok(source)
199    }
200
201    pub fn shebang(&self) -> Option<&'s str> {
202        self.shebang.clone().map(|span| &self.raw[span])
203    }
204
205    pub fn shebang_span(&self) -> Option<Span> {
206        self.shebang.clone()
207    }
208
209    pub fn open_span(&self) -> Option<Span> {
210        self.open.clone()
211    }
212
213    pub fn info(&self) -> Option<&'s str> {
214        self.info.clone().map(|span| &self.raw[span])
215    }
216
217    pub fn info_span(&self) -> Option<Span> {
218        self.info.clone()
219    }
220
221    pub fn frontmatter(&self) -> Option<&'s str> {
222        self.frontmatter.clone().map(|span| &self.raw[span])
223    }
224
225    pub fn frontmatter_span(&self) -> Option<Span> {
226        self.frontmatter.clone()
227    }
228
229    pub fn close_span(&self) -> Option<Span> {
230        self.close.clone()
231    }
232
233    pub fn content(&self) -> &'s str {
234        &self.raw[self.content.clone()]
235    }
236
237    pub fn content_span(&self) -> Span {
238        self.content.clone()
239    }
240}
241
242/// Returns the index after the shebang line, if present
243pub fn strip_shebang(input: &str) -> Option<usize> {
244    // See rust-lang/rust's compiler/rustc_lexer/src/lib.rs's `strip_shebang`
245    // Shebang must start with `#!` literally, without any preceding whitespace.
246    // For simplicity we consider any line starting with `#!` a shebang,
247    // regardless of restrictions put on shebangs by specific platforms.
248    if let Some(rest) = input.strip_prefix("#!") {
249        // Ok, this is a shebang but if the next non-whitespace token is `[`,
250        // then it may be valid Rust code, so consider it Rust code.
251        //
252        // NOTE: rustc considers line and block comments to be whitespace but to avoid
253        // any more awareness of Rust grammar, we are excluding it.
254        if !rest.trim_start().starts_with('[') {
255            // No other choice than to consider this a shebang.
256            let newline_end = input.find('\n').map(|pos| pos + 1).unwrap_or(input.len());
257            return Some(newline_end);
258        }
259    }
260    None
261}
262
263/// Returns the index after any lines with only whitespace, if present
264pub fn strip_ws_lines(input: &str) -> Option<usize> {
265    let ws_end = input.find(|c| !is_whitespace(c)).unwrap_or(input.len());
266    if ws_end == 0 {
267        return None;
268    }
269
270    let nl_start = input[0..ws_end].rfind('\n')?;
271    let nl_end = nl_start + 1;
272    Some(nl_end)
273}
274
275/// True if `c` is considered a whitespace according to Rust language definition.
276/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
277/// for definitions of these classes.
278fn is_whitespace(c: char) -> bool {
279    // This is Pattern_White_Space.
280    //
281    // Note that this set is stable (ie, it doesn't change with different
282    // Unicode versions), so it's ok to just hard-code the values.
283
284    matches!(
285        c,
286        // End-of-line characters
287        | '\u{000A}' // line feed (\n)
288        | '\u{000B}' // vertical tab
289        | '\u{000C}' // form feed
290        | '\u{000D}' // carriage return (\r)
291        | '\u{0085}' // next line (from latin1)
292        | '\u{2028}' // LINE SEPARATOR
293        | '\u{2029}' // PARAGRAPH SEPARATOR
294
295        // `Default_Ignorable_Code_Point` characters
296        | '\u{200E}' // LEFT-TO-RIGHT MARK
297        | '\u{200F}' // RIGHT-TO-LEFT MARK
298
299        // Horizontal space characters
300        | '\u{0009}'   // tab (\t)
301        | '\u{0020}' // space
302    )
303}
304
305/// True if `c` is considered horizontal whitespace according to Rust language definition.
306fn is_horizontal_whitespace(c: char) -> bool {
307    // This is Pattern_White_Space.
308    //
309    // Note that this set is stable (ie, it doesn't change with different
310    // Unicode versions), so it's ok to just hard-code the values.
311
312    matches!(
313        c,
314        // Horizontal space characters
315        '\u{0009}'   // tab (\t)
316        | '\u{0020}' // space
317    )
318}
319
320fn strip_newline(text: &str) -> &str {
321    text.strip_suffix("\r\n")
322        .or_else(|| text.strip_suffix('\n'))
323        .unwrap_or(text)
324}
325
326#[derive(Debug)]
327pub struct FrontmatterError {
328    message: String,
329    primary_span: Span,
330    visible_spans: Vec<Span>,
331}
332
333impl FrontmatterError {
334    pub fn new(message: impl Into<String>, span: Span) -> Self {
335        Self {
336            message: message.into(),
337            primary_span: span,
338            visible_spans: Vec::new(),
339        }
340    }
341
342    pub fn push_visible_span(mut self, span: Span) -> Self {
343        self.visible_spans.push(span);
344        self
345    }
346
347    pub fn message(&self) -> &str {
348        self.message.as_str()
349    }
350
351    pub fn primary_span(&self) -> Span {
352        self.primary_span.clone()
353    }
354
355    pub fn visible_spans(&self) -> &[Span] {
356        &self.visible_spans
357    }
358}
359
360impl std::fmt::Display for FrontmatterError {
361    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
362        self.message.fmt(fmt)
363    }
364}
365
366impl std::error::Error for FrontmatterError {}
367
368#[cfg(test)]
369mod test {
370    use snapbox::assert_data_eq;
371    use snapbox::prelude::*;
372    use snapbox::str;
373
374    use super::*;
375
376    #[track_caller]
377    fn assert_source(source: &str, expected: impl IntoData) {
378        use std::fmt::Write as _;
379
380        let actual = match ScriptSource::parse(source) {
381            Ok(actual) => actual,
382            Err(err) => panic!("unexpected err: {err}"),
383        };
384
385        let mut rendered = String::new();
386        write_optional_field(&mut rendered, "shebang", actual.shebang());
387        write_optional_field(&mut rendered, "info", actual.info());
388        write_optional_field(&mut rendered, "frontmatter", actual.frontmatter());
389        writeln!(&mut rendered, "content: {:?}", actual.content()).unwrap();
390        assert_data_eq!(rendered, expected.raw());
391    }
392
393    fn write_optional_field(writer: &mut dyn std::fmt::Write, field: &str, value: Option<&str>) {
394        if let Some(value) = value {
395            writeln!(writer, "{field}: {value:?}").unwrap();
396        } else {
397            writeln!(writer, "{field}: None").unwrap();
398        }
399    }
400
401    #[track_caller]
402    fn assert_source_err(source: &str, expected: impl IntoData) {
403        match ScriptSource::parse(source) {
404            Ok(d) => panic!("unexpected Ok({d:#?})"),
405            Err(err) => {
406                let report = &[annotate_snippets::Level::ERROR
407                    .primary_title(err.to_string())
408                    .element(
409                        annotate_snippets::Snippet::source(source)
410                            .annotation(
411                                annotate_snippets::AnnotationKind::Primary.span(err.primary_span()),
412                            )
413                            .annotations(err.visible_spans().iter().map(|s| {
414                                annotate_snippets::AnnotationKind::Context.span(s.clone())
415                            })),
416                    )];
417                let renderer = annotate_snippets::Renderer::plain();
418                let actual = renderer.render(report);
419                snapbox::assert_data_eq!(actual, expected.raw())
420            }
421        }
422    }
423
424    #[test]
425    fn split_default() {
426        assert_source(
427            r#"fn main() {}
428"#,
429            str![[r#"
430shebang: None
431info: None
432frontmatter: None
433content: "fn main() {}\n"
434
435"#]],
436        );
437    }
438
439    #[test]
440    fn split_dependencies() {
441        assert_source(
442            r#"---
443[dependencies]
444time="0.1.25"
445---
446fn main() {}
447"#,
448            str![[r#"
449shebang: None
450info: None
451frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
452content: "fn main() {}\n"
453
454"#]],
455        );
456    }
457
458    #[test]
459    fn split_infostring() {
460        assert_source(
461            r#"---cargo
462[dependencies]
463time="0.1.25"
464---
465fn main() {}
466"#,
467            str![[r#"
468shebang: None
469info: "cargo"
470frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
471content: "fn main() {}\n"
472
473"#]],
474        );
475    }
476
477    #[test]
478    fn split_infostring_whitespace() {
479        assert_source(
480            "--- cargo \n\
481[dependencies]\n\
482time=\"0.1.25\"\n\
483---\n\
484fn main() {}\n\
485",
486            str![[r#"
487shebang: None
488info: "cargo"
489frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
490content: "fn main() {}\n"
491
492"#]],
493        );
494    }
495
496    #[test]
497    fn split_shebang() {
498        assert_source(
499            r#"#!/usr/bin/env cargo
500---
501[dependencies]
502time="0.1.25"
503---
504fn main() {}
505"#,
506            str![[r##"
507shebang: "#!/usr/bin/env cargo\n"
508info: None
509frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
510content: "fn main() {}\n"
511
512"##]],
513        );
514    }
515
516    #[test]
517    fn split_crlf() {
518        assert_source(
519            "#!/usr/bin/env cargo\r\n---\r\n[dependencies]\r\ntime=\"0.1.25\"\r\n---\r\nfn main() {}",
520            str![[r##"
521shebang: "#!/usr/bin/env cargo\r\n"
522info: None
523frontmatter: "[dependencies]\r\ntime=\"0.1.25\"\r\n"
524content: "fn main() {}"
525
526"##]],
527        );
528    }
529
530    #[test]
531    fn split_leading_newlines() {
532        assert_source(
533            "#!/usr/bin/env cargo\n\
534    \n\
535\n\
536\n\
537---\n\
538[dependencies]\n\
539time=\"0.1.25\"\n\
540---\n\
541\n\
542\n\
543fn main() {}\n\
544",
545            str![[r##"
546shebang: "#!/usr/bin/env cargo\n"
547info: None
548frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
549content: "\n\nfn main() {}\n"
550
551"##]],
552        );
553    }
554
555    #[test]
556    fn split_attribute() {
557        assert_source(
558            r#"#[allow(dead_code)]
559---
560[dependencies]
561time="0.1.25"
562---
563fn main() {}
564"#,
565            str![[r##"
566shebang: None
567info: None
568frontmatter: None
569content: "#[allow(dead_code)]\n---\n[dependencies]\ntime=\"0.1.25\"\n---\nfn main() {}\n"
570
571"##]],
572        );
573    }
574
575    #[test]
576    fn split_extra_dash() {
577        assert_source(
578            r#"#!/usr/bin/env cargo
579----------
580[dependencies]
581time="0.1.25"
582----------
583
584fn main() {}"#,
585            str![[r##"
586shebang: "#!/usr/bin/env cargo\n"
587info: None
588frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
589content: "\nfn main() {}"
590
591"##]],
592        );
593    }
594
595    #[test]
596    fn split_too_few_dashes() {
597        assert_source_err(
598            r#"#!/usr/bin/env cargo
599--
600[dependencies]
601time="0.1.25"
602--
603fn main() {}
604"#,
605            str![[r#"
606error: found 2 `-` in rust frontmatter, expected at least 3
607  |
6082 | --
609  | --
610...
6116 | fn main() {}
612  |             ^
613"#]],
614        );
615    }
616
617    #[test]
618    fn split_indent() {
619        assert_source(
620            r#"#!/usr/bin/env cargo
621    ---
622    [dependencies]
623    time="0.1.25"
624    ----
625
626fn main() {}
627"#,
628            str![[r##"
629shebang: "#!/usr/bin/env cargo\n"
630info: None
631frontmatter: None
632content: "    ---\n    [dependencies]\n    time=\"0.1.25\"\n    ----\n\nfn main() {}\n"
633
634"##]],
635        );
636    }
637
638    #[test]
639    fn split_escaped() {
640        assert_source(
641            r#"#!/usr/bin/env cargo
642-----
643---
644---
645-----
646
647fn main() {}
648"#,
649            str![[r##"
650shebang: "#!/usr/bin/env cargo\n"
651info: None
652frontmatter: "---\n---\n"
653content: "\nfn main() {}\n"
654
655"##]],
656        );
657    }
658
659    #[test]
660    fn split_invalid_escaped() {
661        assert_source_err(
662            r#"#!/usr/bin/env cargo
663---
664-----
665-----
666---
667
668fn main() {}
669"#,
670            str![[r#"
671error: closing code fence has 2 more `-` than the opening fence
672  |
6732 | ---
674  | ---
6753 | -----
676  |    ^^
677"#]],
678        );
679    }
680
681    #[test]
682    fn split_dashes_in_body() {
683        assert_source(
684            r#"#!/usr/bin/env cargo
685---
686Hello---
687World
688---
689
690fn main() {}
691"#,
692            str![[r##"
693shebang: "#!/usr/bin/env cargo\n"
694info: None
695frontmatter: "Hello---\nWorld\n"
696content: "\nfn main() {}\n"
697
698"##]],
699        );
700    }
701
702    #[test]
703    fn split_mismatched_dashes() {
704        assert_source_err(
705            r#"#!/usr/bin/env cargo
706---
707[dependencies]
708time="0.1.25"
709----
710fn main() {}
711"#,
712            str![[r#"
713error: closing code fence has 1 more `-` than the opening fence
714  |
7152 | ---
716  | ---
717...
7185 | ----
719  |    ^
720"#]],
721        );
722    }
723
724    #[test]
725    fn split_missing_close() {
726        assert_source_err(
727            r#"#!/usr/bin/env cargo
728---
729[dependencies]
730time="0.1.25"
731fn main() {}
732"#,
733            str![[r#"
734error: unclosed frontmatter; expected `---`
735  |
7362 | ---
737  | ---
738...
7395 | fn main() {}
740  |             ^
741"#]],
742        );
743    }
744
745    #[test]
746    fn split_fewer_dashes() {
747        assert_source_err(
748            r#"----
749[dependencies]
750--
751fn main() {}
752"#,
753            str![[r#"
754error: closing code fence has 2 less `-` than the opening fence
755  |
7561 | ----
757  | ----
7582 | [dependencies]
7593 | --
760  | ^^
761"#]],
762        );
763    }
764
765    #[test]
766    fn split_fewer_dashes_by_one() {
767        assert_source_err(
768            r#"----
769[dependencies]
770---
771fn main() {}
772"#,
773            str![[r#"
774error: closing code fence has 1 less `-` than the opening fence
775  |
7761 | ----
777  | ----
7782 | [dependencies]
7793 | ---
780  | ^^^
781"#]],
782        );
783    }
784
785    #[test]
786    fn split_fewer_dashes_before_non_ascii() {
787        // The byte after the short closing fence starts a multi-byte char.
788        assert_source_err(
789            "---
790-\u{2502}
791",
792            str![[r#"
793error: closing code fence has 2 less `-` than the opening fence
794  |
7951 | ---
796  | ---
7972 | -│
798  | ^
799"#]],
800        );
801    }
802}
cargo/util/frontmatter.rs

cargo/util/
frontmatter.rs
