cargo/util/
frontmatter.rs

1type Span = std::ops::Range<usize>;
2
3#[derive(Debug)]
4pub struct ScriptSource<'s> {
5    /// The full file
6    raw: &'s str,
7    /// The `#!/usr/bin/env cargo` line, if present
8    shebang: Option<Span>,
9    /// The code fence opener (`---`)
10    open: Option<Span>,
11    /// Trailing text after `ScriptSource::open` that identifies the meaning of
12    /// `ScriptSource::frontmatter`
13    info: Option<Span>,
14    /// The lines between `ScriptSource::open` and `ScriptSource::close`
15    frontmatter: Option<Span>,
16    /// The code fence closer (`---`)
17    close: Option<Span>,
18    /// All content after the frontmatter and shebang
19    content: Span,
20}
21
22impl<'s> ScriptSource<'s> {
23    pub fn parse(raw: &'s str) -> Result<Self, FrontmatterError> {
24        use winnow::stream::FindSlice as _;
25        use winnow::stream::Location as _;
26        use winnow::stream::Stream as _;
27
28        let content_end = raw.len();
29        let mut source = Self {
30            raw,
31            shebang: None,
32            open: None,
33            info: None,
34            frontmatter: None,
35            close: None,
36            content: 0..content_end,
37        };
38
39        let mut input = winnow::stream::LocatingSlice::new(raw);
40
41        if let Some(shebang_end) = strip_shebang(input.as_ref()) {
42            let shebang_start = input.current_token_start();
43            let _ = input.next_slice(shebang_end);
44            let shebang_end = input.current_token_start();
45            source.shebang = Some(shebang_start..shebang_end);
46            source.content = shebang_end..content_end;
47        }
48
49        // Whitespace may precede a frontmatter but must end with a newline
50        if let Some(nl_end) = strip_ws_lines(input.as_ref()) {
51            let _ = input.next_slice(nl_end);
52        }
53
54        // Opens with a line that starts with 3 or more `-` followed by an optional identifier
55        const FENCE_CHAR: char = '-';
56        let fence_length = input
57            .as_ref()
58            .char_indices()
59            .find_map(|(i, c)| (c != FENCE_CHAR).then_some(i))
60            .unwrap_or_else(|| input.eof_offset());
61        let open_start = input.current_token_start();
62        let fence_pattern = input.next_slice(fence_length);
63        let open_end = input.current_token_start();
64        match fence_length {
65            0 => {
66                return Ok(source);
67            }
68            1 | 2 => {
69                // either not a frontmatter or invalid frontmatter opening
70                return Err(FrontmatterError::new(
71                    format!(
72                        "found {fence_length} `{FENCE_CHAR}` in rust frontmatter, expected at least 3"
73                    ),
74                    raw.len()..raw.len(),
75                ).push_visible_span(open_start..open_end));
76            }
77            _ if u8::try_from(fence_length).is_err() => {
78                return Err(FrontmatterError::new(
79                    format!(
80                        "too many `-` symbols: frontmatter openings may be delimited by up to 255 `-` symbols, but found {fence_length}"
81                    ),
82                    open_start..open_end,
83                ));
84            }
85            _ => {}
86        }
87        source.open = Some(open_start..open_end);
88        let Some(info_nl) = input.find_slice("\n") else {
89            return Err(FrontmatterError::new(
90                format!("unclosed frontmatter; expected `{fence_pattern}`"),
91                raw.len()..raw.len(),
92            )
93            .push_visible_span(open_start..open_end));
94        };
95        let info = input.next_slice(info_nl.start);
96        let info = info.strip_suffix('\r').unwrap_or(info); // already excludes `\n`
97        let info = info.trim_matches(is_horizontal_whitespace);
98        if !info.is_empty() {
99            source.info = Some(raw.substr_range(info).unwrap().into());
100        }
101
102        // Ends with a line that starts with a matching number of `-` only followed by whitespace
103        let nl_fence_pattern = format!("\n{fence_pattern}");
104        let Some(frontmatter_nl) = input.find_slice(nl_fence_pattern.as_str()) else {
105            for prefix_len in (2..=(nl_fence_pattern.len() - 1)).rev() {
106                let Some(frontmatter_nl) = input.find_slice(&nl_fence_pattern[0..prefix_len])
107                else {
108                    continue;
109                };
110                let nl_len = "\n".len();
111                let close_len = prefix_len - nl_len;
112
113                let _ = input.next_slice(frontmatter_nl.start + nl_len);
114                let close_start = input.current_token_start();
115                let _ = input.next_slice(close_len);
116                let close_end = input.current_token_start();
117                let fewer_dashes = fence_length - close_len;
118                return Err(FrontmatterError::new(
119                    format!(
120                        "closing code fence has {fewer_dashes} less `-` than the opening fence"
121                    ),
122                    close_start..close_end,
123                )
124                .push_visible_span(open_start..open_end));
125            }
126            return Err(FrontmatterError::new(
127                format!("unclosed frontmatter; expected `{fence_pattern}`"),
128                raw.len()..raw.len(),
129            )
130            .push_visible_span(open_start..open_end));
131        };
132        let frontmatter_start = input.current_token_start() + 1; // skip nl from infostring
133        let _ = input.next_slice(frontmatter_nl.start + 1);
134        let frontmatter_end = input.current_token_start();
135        source.frontmatter = Some(frontmatter_start..frontmatter_end);
136        let close_start = input.current_token_start();
137        let _ = input.next_slice(fence_length);
138        let close_end = input.current_token_start();
139        source.close = Some(close_start..close_end);
140
141        let nl = input.find_slice("\n");
142        let after_closing_fence = input.next_slice(
143            nl.map(|span| span.end)
144                .unwrap_or_else(|| input.eof_offset()),
145        );
146        let content_start = input.current_token_start();
147        let extra_dashes = after_closing_fence
148            .chars()
149            .take_while(|b| *b == FENCE_CHAR)
150            .count();
151        if 0 < extra_dashes {
152            let extra_start = close_end;
153            let extra_end = extra_start + extra_dashes;
154            return Err(FrontmatterError::new(
155                format!("closing code fence has {extra_dashes} more `-` than the opening fence"),
156                extra_start..extra_end,
157            )
158            .push_visible_span(open_start..open_end));
159        } else {
160            let after_closing_fence = strip_newline(after_closing_fence);
161            let after_closing_fence = after_closing_fence.trim_matches(is_horizontal_whitespace);
162            if !after_closing_fence.is_empty() {
163                // extra characters beyond the original fence pattern
164                return Err(FrontmatterError::new(
165                    format!("unexpected characters after frontmatter close"),
166                    raw.substr_range(after_closing_fence).unwrap().into(),
167                )
168                .push_visible_span(open_start..open_end));
169            }
170        }
171
172        source.content = content_start..content_end;
173
174        if let Some(nl_end) = strip_ws_lines(input.as_ref()) {
175            let _ = input.next_slice(nl_end);
176        }
177        let fence_length = input
178            .as_ref()
179            .char_indices()
180            .find_map(|(i, c)| (c != FENCE_CHAR).then_some(i))
181            .unwrap_or_else(|| input.eof_offset());
182        if 0 < fence_length {
183            let fence_start = input.current_token_start();
184            let fence_end = fence_start + fence_length;
185            return Err(FrontmatterError::new(
186                format!("only one frontmatter is supported"),
187                fence_start..fence_end,
188            )
189            .push_visible_span(open_start..open_end)
190            .push_visible_span(close_start..close_end));
191        }
192
193        Ok(source)
194    }
195
196    pub fn shebang(&self) -> Option<&'s str> {
197        self.shebang.clone().map(|span| &self.raw[span])
198    }
199
200    pub fn shebang_span(&self) -> Option<Span> {
201        self.shebang.clone()
202    }
203
204    pub fn open_span(&self) -> Option<Span> {
205        self.open.clone()
206    }
207
208    pub fn info(&self) -> Option<&'s str> {
209        self.info.clone().map(|span| &self.raw[span])
210    }
211
212    pub fn info_span(&self) -> Option<Span> {
213        self.info.clone()
214    }
215
216    pub fn frontmatter(&self) -> Option<&'s str> {
217        self.frontmatter.clone().map(|span| &self.raw[span])
218    }
219
220    pub fn frontmatter_span(&self) -> Option<Span> {
221        self.frontmatter.clone()
222    }
223
224    pub fn close_span(&self) -> Option<Span> {
225        self.close.clone()
226    }
227
228    pub fn content(&self) -> &'s str {
229        &self.raw[self.content.clone()]
230    }
231
232    pub fn content_span(&self) -> Span {
233        self.content.clone()
234    }
235}
236
237/// Returns the index after the shebang line, if present
238pub fn strip_shebang(input: &str) -> Option<usize> {
239    // See rust-lang/rust's compiler/rustc_lexer/src/lib.rs's `strip_shebang`
240    // Shebang must start with `#!` literally, without any preceding whitespace.
241    // For simplicity we consider any line starting with `#!` a shebang,
242    // regardless of restrictions put on shebangs by specific platforms.
243    if let Some(rest) = input.strip_prefix("#!") {
244        // Ok, this is a shebang but if the next non-whitespace token is `[`,
245        // then it may be valid Rust code, so consider it Rust code.
246        //
247        // NOTE: rustc considers line and block comments to be whitespace but to avoid
248        // any more awareness of Rust grammar, we are excluding it.
249        if !rest.trim_start().starts_with('[') {
250            // No other choice than to consider this a shebang.
251            let newline_end = input.find('\n').map(|pos| pos + 1).unwrap_or(input.len());
252            return Some(newline_end);
253        }
254    }
255    None
256}
257
258/// Returns the index after any lines with only whitespace, if present
259pub fn strip_ws_lines(input: &str) -> Option<usize> {
260    let ws_end = input.find(|c| !is_whitespace(c)).unwrap_or(input.len());
261    if ws_end == 0 {
262        return None;
263    }
264
265    let nl_start = input[0..ws_end].rfind('\n')?;
266    let nl_end = nl_start + 1;
267    Some(nl_end)
268}
269
270/// True if `c` is considered a whitespace according to Rust language definition.
271/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
272/// for definitions of these classes.
273fn is_whitespace(c: char) -> bool {
274    // This is Pattern_White_Space.
275    //
276    // Note that this set is stable (ie, it doesn't change with different
277    // Unicode versions), so it's ok to just hard-code the values.
278
279    matches!(
280        c,
281        // End-of-line characters
282        | '\u{000A}' // line feed (\n)
283        | '\u{000B}' // vertical tab
284        | '\u{000C}' // form feed
285        | '\u{000D}' // carriage return (\r)
286        | '\u{0085}' // next line (from latin1)
287        | '\u{2028}' // LINE SEPARATOR
288        | '\u{2029}' // PARAGRAPH SEPARATOR
289
290        // `Default_Ignorable_Code_Point` characters
291        | '\u{200E}' // LEFT-TO-RIGHT MARK
292        | '\u{200F}' // RIGHT-TO-LEFT MARK
293
294        // Horizontal space characters
295        | '\u{0009}'   // tab (\t)
296        | '\u{0020}' // space
297    )
298}
299
300/// True if `c` is considered horizontal whitespace according to Rust language definition.
301fn is_horizontal_whitespace(c: char) -> bool {
302    // This is Pattern_White_Space.
303    //
304    // Note that this set is stable (ie, it doesn't change with different
305    // Unicode versions), so it's ok to just hard-code the values.
306
307    matches!(
308        c,
309        // Horizontal space characters
310        '\u{0009}'   // tab (\t)
311        | '\u{0020}' // space
312    )
313}
314
315fn strip_newline(text: &str) -> &str {
316    text.strip_suffix("\r\n")
317        .or_else(|| text.strip_suffix('\n'))
318        .unwrap_or(text)
319}
320
321#[derive(Debug)]
322pub struct FrontmatterError {
323    message: String,
324    primary_span: Span,
325    visible_spans: Vec<Span>,
326}
327
328impl FrontmatterError {
329    pub fn new(message: impl Into<String>, span: Span) -> Self {
330        Self {
331            message: message.into(),
332            primary_span: span,
333            visible_spans: Vec::new(),
334        }
335    }
336
337    pub fn push_visible_span(mut self, span: Span) -> Self {
338        self.visible_spans.push(span);
339        self
340    }
341
342    pub fn message(&self) -> &str {
343        self.message.as_str()
344    }
345
346    pub fn primary_span(&self) -> Span {
347        self.primary_span.clone()
348    }
349
350    pub fn visible_spans(&self) -> &[Span] {
351        &self.visible_spans
352    }
353}
354
355impl std::fmt::Display for FrontmatterError {
356    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
357        self.message.fmt(fmt)
358    }
359}
360
361impl std::error::Error for FrontmatterError {}
362
363#[cfg(test)]
364mod test {
365    use snapbox::assert_data_eq;
366    use snapbox::prelude::*;
367    use snapbox::str;
368
369    use super::*;
370
371    #[track_caller]
372    fn assert_source(source: &str, expected: impl IntoData) {
373        use std::fmt::Write as _;
374
375        let actual = match ScriptSource::parse(source) {
376            Ok(actual) => actual,
377            Err(err) => panic!("unexpected err: {err}"),
378        };
379
380        let mut rendered = String::new();
381        write_optional_field(&mut rendered, "shebang", actual.shebang());
382        write_optional_field(&mut rendered, "info", actual.info());
383        write_optional_field(&mut rendered, "frontmatter", actual.frontmatter());
384        writeln!(&mut rendered, "content: {:?}", actual.content()).unwrap();
385        assert_data_eq!(rendered, expected.raw());
386    }
387
388    fn write_optional_field(writer: &mut dyn std::fmt::Write, field: &str, value: Option<&str>) {
389        if let Some(value) = value {
390            writeln!(writer, "{field}: {value:?}").unwrap();
391        } else {
392            writeln!(writer, "{field}: None").unwrap();
393        }
394    }
395
396    #[track_caller]
397    fn assert_source_err(source: &str, expected: impl IntoData) {
398        match ScriptSource::parse(source) {
399            Ok(d) => panic!("unexpected Ok({d:#?})"),
400            Err(err) => {
401                let report = &[annotate_snippets::Level::ERROR
402                    .primary_title(err.to_string())
403                    .element(
404                        annotate_snippets::Snippet::source(source)
405                            .annotation(
406                                annotate_snippets::AnnotationKind::Primary.span(err.primary_span()),
407                            )
408                            .annotations(err.visible_spans().iter().map(|s| {
409                                annotate_snippets::AnnotationKind::Context.span(s.clone())
410                            })),
411                    )];
412                let renderer = annotate_snippets::Renderer::plain();
413                let actual = renderer.render(report);
414                snapbox::assert_data_eq!(actual, expected.raw())
415            }
416        }
417    }
418
419    #[test]
420    fn split_default() {
421        assert_source(
422            r#"fn main() {}
423"#,
424            str![[r#"
425shebang: None
426info: None
427frontmatter: None
428content: "fn main() {}\n"
429
430"#]],
431        );
432    }
433
434    #[test]
435    fn split_dependencies() {
436        assert_source(
437            r#"---
438[dependencies]
439time="0.1.25"
440---
441fn main() {}
442"#,
443            str![[r#"
444shebang: None
445info: None
446frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
447content: "fn main() {}\n"
448
449"#]],
450        );
451    }
452
453    #[test]
454    fn split_infostring() {
455        assert_source(
456            r#"---cargo
457[dependencies]
458time="0.1.25"
459---
460fn main() {}
461"#,
462            str![[r#"
463shebang: None
464info: "cargo"
465frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
466content: "fn main() {}\n"
467
468"#]],
469        );
470    }
471
472    #[test]
473    fn split_infostring_whitespace() {
474        assert_source(
475            "--- cargo \n\
476[dependencies]\n\
477time=\"0.1.25\"\n\
478---\n\
479fn main() {}\n\
480",
481            str![[r#"
482shebang: None
483info: "cargo"
484frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
485content: "fn main() {}\n"
486
487"#]],
488        );
489    }
490
491    #[test]
492    fn split_shebang() {
493        assert_source(
494            r#"#!/usr/bin/env cargo
495---
496[dependencies]
497time="0.1.25"
498---
499fn main() {}
500"#,
501            str![[r##"
502shebang: "#!/usr/bin/env cargo\n"
503info: None
504frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
505content: "fn main() {}\n"
506
507"##]],
508        );
509    }
510
511    #[test]
512    fn split_crlf() {
513        assert_source(
514            "#!/usr/bin/env cargo\r\n---\r\n[dependencies]\r\ntime=\"0.1.25\"\r\n---\r\nfn main() {}",
515            str![[r##"
516shebang: "#!/usr/bin/env cargo\r\n"
517info: None
518frontmatter: "[dependencies]\r\ntime=\"0.1.25\"\r\n"
519content: "fn main() {}"
520
521"##]],
522        );
523    }
524
525    #[test]
526    fn split_leading_newlines() {
527        assert_source(
528            "#!/usr/bin/env cargo\n\
529    \n\
530\n\
531\n\
532---\n\
533[dependencies]\n\
534time=\"0.1.25\"\n\
535---\n\
536\n\
537\n\
538fn main() {}\n\
539",
540            str![[r##"
541shebang: "#!/usr/bin/env cargo\n"
542info: None
543frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
544content: "\n\nfn main() {}\n"
545
546"##]],
547        );
548    }
549
550    #[test]
551    fn split_attribute() {
552        assert_source(
553            r#"#[allow(dead_code)]
554---
555[dependencies]
556time="0.1.25"
557---
558fn main() {}
559"#,
560            str![[r##"
561shebang: None
562info: None
563frontmatter: None
564content: "#[allow(dead_code)]\n---\n[dependencies]\ntime=\"0.1.25\"\n---\nfn main() {}\n"
565
566"##]],
567        );
568    }
569
570    #[test]
571    fn split_extra_dash() {
572        assert_source(
573            r#"#!/usr/bin/env cargo
574----------
575[dependencies]
576time="0.1.25"
577----------
578
579fn main() {}"#,
580            str![[r##"
581shebang: "#!/usr/bin/env cargo\n"
582info: None
583frontmatter: "[dependencies]\ntime=\"0.1.25\"\n"
584content: "\nfn main() {}"
585
586"##]],
587        );
588    }
589
590    #[test]
591    fn split_too_few_dashes() {
592        assert_source_err(
593            r#"#!/usr/bin/env cargo
594--
595[dependencies]
596time="0.1.25"
597--
598fn main() {}
599"#,
600            str![[r#"
601error: found 2 `-` in rust frontmatter, expected at least 3
602  |
6032 | --
604  | --
605...
6066 | fn main() {}
607  |             ^
608"#]],
609        );
610    }
611
612    #[test]
613    fn split_indent() {
614        assert_source(
615            r#"#!/usr/bin/env cargo
616    ---
617    [dependencies]
618    time="0.1.25"
619    ----
620
621fn main() {}
622"#,
623            str![[r##"
624shebang: "#!/usr/bin/env cargo\n"
625info: None
626frontmatter: None
627content: "    ---\n    [dependencies]\n    time=\"0.1.25\"\n    ----\n\nfn main() {}\n"
628
629"##]],
630        );
631    }
632
633    #[test]
634    fn split_escaped() {
635        assert_source(
636            r#"#!/usr/bin/env cargo
637-----
638---
639---
640-----
641
642fn main() {}
643"#,
644            str![[r##"
645shebang: "#!/usr/bin/env cargo\n"
646info: None
647frontmatter: "---\n---\n"
648content: "\nfn main() {}\n"
649
650"##]],
651        );
652    }
653
654    #[test]
655    fn split_invalid_escaped() {
656        assert_source_err(
657            r#"#!/usr/bin/env cargo
658---
659-----
660-----
661---
662
663fn main() {}
664"#,
665            str![[r#"
666error: closing code fence has 2 more `-` than the opening fence
667  |
6682 | ---
669  | ---
6703 | -----
671  |    ^^
672"#]],
673        );
674    }
675
676    #[test]
677    fn split_dashes_in_body() {
678        assert_source(
679            r#"#!/usr/bin/env cargo
680---
681Hello---
682World
683---
684
685fn main() {}
686"#,
687            str![[r##"
688shebang: "#!/usr/bin/env cargo\n"
689info: None
690frontmatter: "Hello---\nWorld\n"
691content: "\nfn main() {}\n"
692
693"##]],
694        );
695    }
696
697    #[test]
698    fn split_mismatched_dashes() {
699        assert_source_err(
700            r#"#!/usr/bin/env cargo
701---
702[dependencies]
703time="0.1.25"
704----
705fn main() {}
706"#,
707            str![[r#"
708error: closing code fence has 1 more `-` than the opening fence
709  |
7102 | ---
711  | ---
712...
7135 | ----
714  |    ^
715"#]],
716        );
717    }
718
719    #[test]
720    fn split_missing_close() {
721        assert_source_err(
722            r#"#!/usr/bin/env cargo
723---
724[dependencies]
725time="0.1.25"
726fn main() {}
727"#,
728            str![[r#"
729error: unclosed frontmatter; expected `---`
730  |
7312 | ---
732  | ---
733...
7345 | fn main() {}
735  |             ^
736"#]],
737        );
738    }
739
740    #[test]
741    fn split_fewer_dashes() {
742        assert_source_err(
743            r#"----
744[dependencies]
745--
746fn main() {}
747"#,
748            str![[r#"
749error: closing code fence has 2 less `-` than the opening fence
750  |
7511 | ----
752  | ----
7532 | [dependencies]
7543 | --
755  | ^^
756"#]],
757        );
758    }
759
760    #[test]
761    fn split_fewer_dashes_by_one() {
762        assert_source_err(
763            r#"----
764[dependencies]
765---
766fn main() {}
767"#,
768            str![[r#"
769error: closing code fence has 1 less `-` than the opening fence
770  |
7711 | ----
772  | ----
7732 | [dependencies]
7743 | ---
775  | ^^^
776"#]],
777        );
778    }
779
780    #[test]
781    fn split_fewer_dashes_before_non_ascii() {
782        // The byte after the short closing fence starts a multi-byte char.
783        assert_source_err(
784            "---
785-\u{2502}
786",
787            str![[r#"
788error: closing code fence has 2 less `-` than the opening fence
789  |
7901 | ---
791  | ---
7922 | -│
793  | ^
794"#]],
795        );
796    }
797}
cargo/util/frontmatter.rs

cargo/util/
frontmatter.rs
