rustc_parse/lexer/
unicode_chars.rs

Help
1//! Characters and their corresponding confusables were collected from
2//! <https://www.unicode.org/Public/security/10.0.0/confusables.txt>
3
4use rustc_span::{BytePos, Pos, Span, kw};
5
6use super::Lexer;
7use crate::errors::TokenSubstitution;
8use crate::token::{self, Delimiter};
9
10#[rustfmt::skip] // for line breaks
11pub(super) static UNICODE_ARRAY: &[(char, &str, &str)] = &[
12    (' ', "Line Separator", " "),
13    (' ', "Paragraph Separator", " "),
14    (' ', "Ogham Space mark", " "),
15    (' ', "En Quad", " "),
16    (' ', "Em Quad", " "),
17    (' ', "En Space", " "),
18    (' ', "Em Space", " "),
19    (' ', "Three-Per-Em Space", " "),
20    (' ', "Four-Per-Em Space", " "),
21    (' ', "Six-Per-Em Space", " "),
22    (' ', "Punctuation Space", " "),
23    (' ', "Thin Space", " "),
24    (' ', "Hair Space", " "),
25    (' ', "Medium Mathematical Space", " "),
26    (' ', "No-Break Space", " "),
27    (' ', "Figure Space", " "),
28    (' ', "Narrow No-Break Space", " "),
29    ('　', "Ideographic Space", " "),
30
31    ('ߺ', "Nko Lajanyalan", "_"),
32    ('﹍', "Dashed Low Line", "_"),
33    ('﹎', "Centreline Low Line", "_"),
34    ('﹏', "Wavy Low Line", "_"),
35    ('＿', "Fullwidth Low Line", "_"),
36
37    ('‐', "Hyphen", "-"),
38    ('‑', "Non-Breaking Hyphen", "-"),
39    ('‒', "Figure Dash", "-"),
40    ('–', "En Dash", "-"),
41    ('—', "Em Dash", "-"),
42    ('﹘', "Small Em Dash", "-"),
43    ('۔', "Arabic Full Stop", "-"),
44    ('⁃', "Hyphen Bullet", "-"),
45    ('˗', "Modifier Letter Minus Sign", "-"),
46    ('−', "Minus Sign", "-"),
47    ('➖', "Heavy Minus Sign", "-"),
48    ('Ⲻ', "Coptic Letter Dialect-P Ni", "-"),
49    ('ー', "Katakana-Hiragana Prolonged Sound Mark", "-"),
50    ('－', "Fullwidth Hyphen-Minus", "-"),
51    ('―', "Horizontal Bar", "-"),
52    ('─', "Box Drawings Light Horizontal", "-"),
53    ('━', "Box Drawings Heavy Horizontal", "-"),
54    ('㇐', "CJK Stroke H", "-"),
55    ('ꟷ', "Latin Epigraphic Letter Sideways I", "-"),
56    ('ᅳ', "Hangul Jungseong Eu", "-"),
57    ('ㅡ', "Hangul Letter Eu", "-"),
58    ('一', "CJK Unified Ideograph-4E00", "-"),
59    ('⼀', "Kangxi Radical One", "-"),
60
61    ('؍', "Arabic Date Separator", ","),
62    ('٫', "Arabic Decimal Separator", ","),
63    ('‚', "Single Low-9 Quotation Mark", ","),
64    ('¸', "Cedilla", ","),
65    ('ꓹ', "Lisu Letter Tone Na Po", ","),
66    ('，', "Fullwidth Comma", ","),
67
68    (';', "Greek Question Mark", ";"),
69    ('；', "Fullwidth Semicolon", ";"),
70    ('︔', "Presentation Form For Vertical Semicolon", ";"),
71
72    ('ः', "Devanagari Sign Visarga", ":"),
73    ('ઃ', "Gujarati Sign Visarga", ":"),
74    ('：', "Fullwidth Colon", ":"),
75    ('։', "Armenian Full Stop", ":"),
76    ('܃', "Syriac Supralinear Colon", ":"),
77    ('܄', "Syriac Sublinear Colon", ":"),
78    ('᛬', "Runic Multiple Punctuation", ":"),
79    ('︰', "Presentation Form For Vertical Two Dot Leader", ":"),
80    ('᠃', "Mongolian Full Stop", ":"),
81    ('᠉', "Mongolian Manchu Full Stop", ":"),
82    ('⁚', "Two Dot Punctuation", ":"),
83    ('׃', "Hebrew Punctuation Sof Pasuq", ":"),
84    ('˸', "Modifier Letter Raised Colon", ":"),
85    ('꞉', "Modifier Letter Colon", ":"),
86    ('∶', "Ratio", ":"),
87    ('ː', "Modifier Letter Triangular Colon", ":"),
88    ('ꓽ', "Lisu Letter Tone Mya Jeu", ":"),
89    ('︓', "Presentation Form For Vertical Colon", ":"),
90
91    ('！', "Fullwidth Exclamation Mark", "!"),
92    ('ǃ', "Latin Letter Retroflex Click", "!"),
93    ('ⵑ', "Tifinagh Letter Tuareg Yang", "!"),
94    ('︕', "Presentation Form For Vertical Exclamation Mark", "!"),
95
96    ('ʔ', "Latin Letter Glottal Stop", "?"),
97    ('Ɂ', "Latin Capital Letter Glottal Stop", "?"),
98    ('ॽ', "Devanagari Letter Glottal Stop", "?"),
99    ('Ꭾ', "Cherokee Letter He", "?"),
100    ('ꛫ', "Bamum Letter Ntuu", "?"),
101    ('？', "Fullwidth Question Mark", "?"),
102    ('︖', "Presentation Form For Vertical Question Mark", "?"),
103
104    ('𝅭', "Musical Symbol Combining Augmentation Dot", "."),
105    ('․', "One Dot Leader", "."),
106    ('܁', "Syriac Supralinear Full Stop", "."),
107    ('܂', "Syriac Sublinear Full Stop", "."),
108    ('꘎', "Vai Full Stop", "."),
109    ('𐩐', "Kharoshthi Punctuation Dot", "."),
110    ('٠', "Arabic-Indic Digit Zero", "."),
111    ('۰', "Extended Arabic-Indic Digit Zero", "."),
112    ('ꓸ', "Lisu Letter Tone Mya Ti", "."),
113    ('·', "Middle Dot", "."),
114    ('・', "Katakana Middle Dot", "."),
115    ('･', "Halfwidth Katakana Middle Dot", "."),
116    ('᛫', "Runic Single Punctuation", "."),
117    ('·', "Greek Ano Teleia", "."),
118    ('⸱', "Word Separator Middle Dot", "."),
119    ('𐄁', "Aegean Word Separator Dot", "."),
120    ('•', "Bullet", "."),
121    ('‧', "Hyphenation Point", "."),
122    ('∙', "Bullet Operator", "."),
123    ('⋅', "Dot Operator", "."),
124    ('ꞏ', "Latin Letter Sinological Dot", "."),
125    ('ᐧ', "Canadian Syllabics Final Middle Dot", "."),
126    ('ᐧ', "Canadian Syllabics Final Middle Dot", "."),
127    ('．', "Fullwidth Full Stop", "."),
128    ('。', "Ideographic Full Stop", "."),
129    ('︒', "Presentation Form For Vertical Ideographic Full Stop", "."),
130
131    ('՝', "Armenian Comma", "'"),
132    ('＇', "Fullwidth Apostrophe", "'"),
133    ('‘', "Left Single Quotation Mark", "'"),
134    ('’', "Right Single Quotation Mark", "'"),
135    ('‛', "Single High-Reversed-9 Quotation Mark", "'"),
136    ('′', "Prime", "'"),
137    ('‵', "Reversed Prime", "'"),
138    ('՚', "Armenian Apostrophe", "'"),
139    ('׳', "Hebrew Punctuation Geresh", "'"),
140    ('`', "Grave Accent", "'"),
141    ('`', "Greek Varia", "'"),
142    ('｀', "Fullwidth Grave Accent", "'"),
143    ('´', "Acute Accent", "'"),
144    ('΄', "Greek Tonos", "'"),
145    ('´', "Greek Oxia", "'"),
146    ('᾽', "Greek Koronis", "'"),
147    ('᾿', "Greek Psili", "'"),
148    ('῾', "Greek Dasia", "'"),
149    ('ʹ', "Modifier Letter Prime", "'"),
150    ('ʹ', "Greek Numeral Sign", "'"),
151    ('ˈ', "Modifier Letter Vertical Line", "'"),
152    ('ˊ', "Modifier Letter Acute Accent", "'"),
153    ('ˋ', "Modifier Letter Grave Accent", "'"),
154    ('˴', "Modifier Letter Middle Grave Accent", "'"),
155    ('ʻ', "Modifier Letter Turned Comma", "'"),
156    ('ʽ', "Modifier Letter Reversed Comma", "'"),
157    ('ʼ', "Modifier Letter Apostrophe", "'"),
158    ('ʾ', "Modifier Letter Right Half Ring", "'"),
159    ('ꞌ', "Latin Small Letter Saltillo", "'"),
160    ('י', "Hebrew Letter Yod", "'"),
161    ('ߴ', "Nko High Tone Apostrophe", "'"),
162    ('ߵ', "Nko Low Tone Apostrophe", "'"),
163    ('ᑊ', "Canadian Syllabics West-Cree P", "'"),
164    ('ᛌ', "Runic Letter Short-Twig-Sol S", "'"),
165    ('𖽑', "Miao Sign Aspiration", "'"),
166    ('𖽒', "Miao Sign Reformed Voicing", "'"),
167
168    ('᳓', "Vedic Sign Nihshvasa", "\""),
169    ('＂', "Fullwidth Quotation Mark", "\""),
170    ('“', "Left Double Quotation Mark", "\""),
171    ('”', "Right Double Quotation Mark", "\""),
172    ('‟', "Double High-Reversed-9 Quotation Mark", "\""),
173    ('″', "Double Prime", "\""),
174    ('‶', "Reversed Double Prime", "\""),
175    ('〃', "Ditto Mark", "\""),
176    ('״', "Hebrew Punctuation Gershayim", "\""),
177    ('˝', "Double Acute Accent", "\""),
178    ('ʺ', "Modifier Letter Double Prime", "\""),
179    ('˶', "Modifier Letter Middle Double Acute Accent", "\""),
180    ('˵', "Modifier Letter Middle Double Grave Accent", "\""),
181    ('ˮ', "Modifier Letter Double Apostrophe", "\""),
182    ('ײ', "Hebrew Ligature Yiddish Double Yod", "\""),
183    ('❞', "Heavy Double Comma Quotation Mark Ornament", "\""),
184    ('❝', "Heavy Double Turned Comma Quotation Mark Ornament", "\""),
185
186    ('（', "Fullwidth Left Parenthesis", "("),
187    ('❨', "Medium Left Parenthesis Ornament", "("),
188    ('﴾', "Ornate Left Parenthesis", "("),
189
190    ('）', "Fullwidth Right Parenthesis", ")"),
191    ('❩', "Medium Right Parenthesis Ornament", ")"),
192    ('﴿', "Ornate Right Parenthesis", ")"),
193
194    ('［', "Fullwidth Left Square Bracket", "["),
195    ('❲', "Light Left Tortoise Shell Bracket Ornament", "["),
196    ('「', "Left Corner Bracket", "["),
197    ('『', "Left White Corner Bracket", "["),
198    ('【', "Left Black Lenticular Bracket", "["),
199    ('〔', "Left Tortoise Shell Bracket", "["),
200    ('〖', "Left White Lenticular Bracket", "["),
201    ('〘', "Left White Tortoise Shell Bracket", "["),
202    ('〚', "Left White Square Bracket", "["),
203
204    ('］', "Fullwidth Right Square Bracket", "]"),
205    ('❳', "Light Right Tortoise Shell Bracket Ornament", "]"),
206    ('」', "Right Corner Bracket", "]"),
207    ('』', "Right White Corner Bracket", "]"),
208    ('】', "Right Black Lenticular Bracket", "]"),
209    ('〕', "Right Tortoise Shell Bracket", "]"),
210    ('〗', "Right White Lenticular Bracket", "]"),
211    ('〙', "Right White Tortoise Shell Bracket", "]"),
212    ('〛', "Right White Square Bracket", "]"),
213
214    ('❴', "Medium Left Curly Bracket Ornament", "{"),
215    ('𝄔', "Musical Symbol Brace", "{"),
216    ('｛', "Fullwidth Left Curly Bracket", "{"),
217
218    ('❵', "Medium Right Curly Bracket Ornament", "}"),
219    ('｝', "Fullwidth Right Curly Bracket", "}"),
220
221    ('⁎', "Low Asterisk", "*"),
222    ('٭', "Arabic Five Pointed Star", "*"),
223    ('∗', "Asterisk Operator", "*"),
224    ('𐌟', "Old Italic Letter Ess", "*"),
225    ('＊', "Fullwidth Asterisk", "*"),
226
227    ('᜵', "Philippine Single Punctuation", "/"),
228    ('⁁', "Caret Insertion Point", "/"),
229    ('∕', "Division Slash", "/"),
230    ('⁄', "Fraction Slash", "/"),
231    ('╱', "Box Drawings Light Diagonal Upper Right To Lower Left", "/"),
232    ('⟋', "Mathematical Rising Diagonal", "/"),
233    ('⧸', "Big Solidus", "/"),
234    ('𝈺', "Greek Instrumental Notation Symbol-47", "/"),
235    ('㇓', "CJK Stroke Sp", "/"),
236    ('〳', "Vertical Kana Repeat Mark Upper Half", "/"),
237    ('Ⳇ', "Coptic Capital Letter Old Coptic Esh", "/"),
238    ('ノ', "Katakana Letter No", "/"),
239    ('丿', "CJK Unified Ideograph-4E3F", "/"),
240    ('⼃', "Kangxi Radical Slash", "/"),
241    ('／', "Fullwidth Solidus", "/"),
242
243    ('＼', "Fullwidth Reverse Solidus", "\\"),
244    ('﹨', "Small Reverse Solidus", "\\"),
245    ('∖', "Set Minus", "\\"),
246    ('⟍', "Mathematical Falling Diagonal", "\\"),
247    ('⧵', "Reverse Solidus Operator", "\\"),
248    ('⧹', "Big Reverse Solidus", "\\"),
249    ('⧹', "Greek Vocal Notation Symbol-16", "\\"),
250    ('⧹', "Greek Instrumental Symbol-48", "\\"),
251    ('㇔', "CJK Stroke D", "\\"),
252    ('丶', "CJK Unified Ideograph-4E36", "\\"),
253    ('⼂', "Kangxi Radical Dot", "\\"),
254    ('、', "Ideographic Comma", "\\"),
255    ('ヽ', "Katakana Iteration Mark", "\\"),
256
257    ('ꝸ', "Latin Small Letter Um", "&"),
258    ('＆', "Fullwidth Ampersand", "&"),
259
260    ('᛭', "Runic Cross Punctuation", "+"),
261    ('➕', "Heavy Plus Sign", "+"),
262    ('𐊛', "Lycian Letter H", "+"),
263    ('﬩', "Hebrew Letter Alternative Plus Sign", "+"),
264    ('＋', "Fullwidth Plus Sign", "+"),
265
266    ('‹', "Single Left-Pointing Angle Quotation Mark", "<"),
267    ('❮', "Heavy Left-Pointing Angle Quotation Mark Ornament", "<"),
268    ('˂', "Modifier Letter Left Arrowhead", "<"),
269    ('𝈶', "Greek Instrumental Symbol-40", "<"),
270    ('ᐸ', "Canadian Syllabics Pa", "<"),
271    ('ᚲ', "Runic Letter Kauna", "<"),
272    ('❬', "Medium Left-Pointing Angle Bracket Ornament", "<"),
273    ('⟨', "Mathematical Left Angle Bracket", "<"),
274    ('〈', "Left-Pointing Angle Bracket", "<"),
275    ('〈', "Left Angle Bracket", "<"),
276    ('㇛', "CJK Stroke Pd", "<"),
277    ('く', "Hiragana Letter Ku", "<"),
278    ('𡿨', "CJK Unified Ideograph-21FE8", "<"),
279    ('《', "Left Double Angle Bracket", "<"),
280    ('＜', "Fullwidth Less-Than Sign", "<"),
281
282    ('᐀', "Canadian Syllabics Hyphen", "="),
283    ('⹀', "Double Hyphen", "="),
284    ('゠', "Katakana-Hiragana Double Hyphen", "="),
285    ('꓿', "Lisu Punctuation Full Stop", "="),
286    ('＝', "Fullwidth Equals Sign", "="),
287
288    ('›', "Single Right-Pointing Angle Quotation Mark", ">"),
289    ('❯', "Heavy Right-Pointing Angle Quotation Mark Ornament", ">"),
290    ('˃', "Modifier Letter Right Arrowhead", ">"),
291    ('𝈷', "Greek Instrumental Symbol-42", ">"),
292    ('ᐳ', "Canadian Syllabics Po", ">"),
293    ('𖼿', "Miao Letter Archaic Zza", ">"),
294    ('❭', "Medium Right-Pointing Angle Bracket Ornament", ">"),
295    ('⟩', "Mathematical Right Angle Bracket", ">"),
296    ('〉', "Right-Pointing Angle Bracket", ">"),
297    ('〉', "Right Angle Bracket", ">"),
298    ('》', "Right Double Angle Bracket", ">"),
299    ('＞', "Fullwidth Greater-Than Sign", ">"),
300
301    ('⩵', "Two Consecutive Equals Signs", "==")
302];
303
304// FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs, instead of
305// keeping the substitution token in this table. Ideally, this should be inside `rustc_lexer`.
306// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add
307// fancier error recovery to it, as there will be less overall work to do this way.
308const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
309    (" ", "Space", None),
310    ("_", "Underscore", Some(token::Ident(kw::Underscore, token::IdentIsRaw::No))),
311    ("-", "Minus/Hyphen", Some(token::BinOp(token::Minus))),
312    (",", "Comma", Some(token::Comma)),
313    (";", "Semicolon", Some(token::Semi)),
314    (":", "Colon", Some(token::Colon)),
315    ("!", "Exclamation Mark", Some(token::Not)),
316    ("?", "Question Mark", Some(token::Question)),
317    (".", "Period", Some(token::Dot)),
318    ("(", "Left Parenthesis", Some(token::OpenDelim(Delimiter::Parenthesis))),
319    (")", "Right Parenthesis", Some(token::CloseDelim(Delimiter::Parenthesis))),
320    ("[", "Left Square Bracket", Some(token::OpenDelim(Delimiter::Bracket))),
321    ("]", "Right Square Bracket", Some(token::CloseDelim(Delimiter::Bracket))),
322    ("{", "Left Curly Brace", Some(token::OpenDelim(Delimiter::Brace))),
323    ("}", "Right Curly Brace", Some(token::CloseDelim(Delimiter::Brace))),
324    ("*", "Asterisk", Some(token::BinOp(token::Star))),
325    ("/", "Slash", Some(token::BinOp(token::Slash))),
326    ("\\", "Backslash", None),
327    ("&", "Ampersand", Some(token::BinOp(token::And))),
328    ("+", "Plus Sign", Some(token::BinOp(token::Plus))),
329    ("<", "Less-Than Sign", Some(token::Lt)),
330    ("=", "Equals Sign", Some(token::Eq)),
331    ("==", "Double Equals Sign", Some(token::EqEq)),
332    (">", "Greater-Than Sign", Some(token::Gt)),
333    // FIXME: Literals are already lexed by this point, so we can't recover gracefully just by
334    // spitting the correct token out.
335    ("'", "Single Quote", None),
336    ("\"", "Quotation Mark", None),
337];
338
339pub(super) fn check_for_substitution(
340    lexer: &Lexer<'_, '_>,
341    pos: BytePos,
342    ch: char,
343    count: usize,
344) -> (Option<token::TokenKind>, Option<TokenSubstitution>) {
345    let Some(&(_, u_name, ascii_str)) = UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch) else {
346        return (None, None);
347    };
348
349    let span = Span::with_root_ctxt(pos, pos + Pos::from_usize(ch.len_utf8() * count));
350
351    let Some((_, ascii_name, token)) = ASCII_ARRAY.iter().find(|&&(s, _, _)| s == ascii_str) else {
352        let msg = format!("substitution character not found for '{ch}'");
353        lexer.dcx().span_bug(span, msg);
354    };
355
356    // special help suggestion for "directed" double quotes
357    let sugg = if let Some(s) = peek_delimited(&lexer.src[lexer.src_index(pos)..], '“', '”') {
358        let span = Span::with_root_ctxt(
359            pos,
360            pos + Pos::from_usize('“'.len_utf8() + s.len() + '”'.len_utf8()),
361        );
362        Some(TokenSubstitution::DirectedQuotes {
363            span,
364            suggestion: format!("\"{s}\""),
365            ascii_str,
366            ascii_name,
367        })
368    } else {
369        let suggestion = ascii_str.to_string().repeat(count);
370        Some(TokenSubstitution::Other {
371            span,
372            suggestion,
373            ch: ch.to_string(),
374            u_name,
375            ascii_str,
376            ascii_name,
377        })
378    };
379    (token.clone(), sugg)
380}
381
382/// Extract string if found at current position with given delimiters
383fn peek_delimited(text: &str, from_ch: char, to_ch: char) -> Option<&str> {
384    let mut chars = text.chars();
385    let first_char = chars.next()?;
386    if first_char != from_ch {
387        return None;
388    }
389    let last_char_idx = chars.as_str().find(to_ch)?;
390    Some(&chars.as_str()[..last_char_idx])
391}
rustc_parse/lexer/unicode_chars.rs

rustc_parse/lexer/
unicode_chars.rs
