tidy/
ui_tests.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
//! Tidy check to ensure below in UI test directories:
//! - the number of entries in each directory must be less than `ENTRY_LIMIT`
//! - there are no stray `.stderr` files

use std::collections::{BTreeSet, HashMap};
use std::ffi::OsStr;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};

use ignore::Walk;

// FIXME: GitHub's UI truncates file lists that exceed 1000 entries, so these
// should all be 1000 or lower. Limits significantly smaller than 1000 are also
// desirable, because large numbers of files are unwieldy in general. See issue
// #73494.
const ENTRY_LIMIT: u32 = 901;
// FIXME: The following limits should be reduced eventually.

const ISSUES_ENTRY_LIMIT: u32 = 1672;

const EXPECTED_TEST_FILE_EXTENSIONS: &[&str] = &[
    "rs",     // test source files
    "stderr", // expected stderr file, corresponds to a rs file
    "svg",    // expected svg file, corresponds to a rs file, equivalent to stderr
    "stdout", // expected stdout file, corresponds to a rs file
    "fixed",  // expected source file after applying fixes
    "md",     // test directory descriptions
    "ftl",    // translation tests
];

const EXTENSION_EXCEPTION_PATHS: &[&str] = &[
    "tests/ui/asm/named-asm-labels.s", // loading an external asm file to test named labels lint
    "tests/ui/codegen/mismatched-data-layout.json", // testing mismatched data layout w/ custom targets
    "tests/ui/check-cfg/my-awesome-platform.json",  // testing custom targets with cfgs
    "tests/ui/argfile/commandline-argfile-badutf8.args", // passing args via a file
    "tests/ui/argfile/commandline-argfile.args",    // passing args via a file
    "tests/ui/crate-loading/auxiliary/libfoo.rlib", // testing loading a manually created rlib
    "tests/ui/include-macros/data.bin", // testing including data with the include macros
    "tests/ui/include-macros/file.txt", // testing including data with the include macros
    "tests/ui/macros/macro-expanded-include/file.txt", // testing including data with the include macros
    "tests/ui/macros/not-utf8.bin", // testing including data with the include macros
    "tests/ui/macros/syntax-extension-source-utils-files/includeme.fragment", // more include
    "tests/ui/proc-macro/auxiliary/included-file.txt", // more include
    "tests/ui/unpretty/auxiliary/data.txt", // more include
    "tests/ui/invalid/foo.natvis.xml", // sample debugger visualizer
    "tests/ui/sanitizer/dataflow-abilist.txt", // dataflow sanitizer ABI list file
    "tests/ui/shell-argfiles/shell-argfiles.args", // passing args via a file
    "tests/ui/shell-argfiles/shell-argfiles-badquotes.args", // passing args via a file
    "tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args", // passing args via a file
    "tests/ui/shell-argfiles/shell-argfiles-via-argfile.args", // passing args via a file
    "tests/ui/std/windows-bat-args1.bat", // tests escaping arguments through batch files
    "tests/ui/std/windows-bat-args2.bat", // tests escaping arguments through batch files
    "tests/ui/std/windows-bat-args3.bat", // tests escaping arguments through batch files
];

fn check_entries(tests_path: &Path, bad: &mut bool) {
    let mut directories: HashMap<PathBuf, u32> = HashMap::new();

    for dir in Walk::new(&tests_path.join("ui")) {
        if let Ok(entry) = dir {
            let parent = entry.path().parent().unwrap().to_path_buf();
            *directories.entry(parent).or_default() += 1;
        }
    }

    let (mut max, mut max_issues) = (0, 0);
    for (dir_path, count) in directories {
        let is_issues_dir = tests_path.join("ui/issues") == dir_path;
        let (limit, maxcnt) = if is_issues_dir {
            (ISSUES_ENTRY_LIMIT, &mut max_issues)
        } else {
            (ENTRY_LIMIT, &mut max)
        };
        *maxcnt = (*maxcnt).max(count);
        if count > limit {
            tidy_error!(
                bad,
                "following path contains more than {} entries, \
                    you should move the test to some relevant subdirectory (current: {}): {}",
                limit,
                count,
                dir_path.display()
            );
        }
    }
    if ISSUES_ENTRY_LIMIT > max_issues {
        tidy_error!(
            bad,
            "`ISSUES_ENTRY_LIMIT` is too high (is {ISSUES_ENTRY_LIMIT}, should be {max_issues})"
        );
    }
}

pub fn check(root_path: &Path, bless: bool, bad: &mut bool) {
    let issues_txt_header = r#"============================================================
    ⚠️⚠️⚠️NOTHING SHOULD EVER BE ADDED TO THIS LIST⚠️⚠️⚠️
============================================================
"#;

    let path = &root_path.join("tests");
    check_entries(&path, bad);

    // the list of files in ui tests that are allowed to start with `issue-XXXX`
    // BTreeSet because we would like a stable ordering so --bless works
    let mut prev_line = "";
    let mut is_sorted = true;
    let allowed_issue_names: BTreeSet<_> = include_str!("issues.txt")
        .strip_prefix(issues_txt_header)
        .unwrap()
        .lines()
        .map(|line| {
            if prev_line > line {
                is_sorted = false;
            }

            prev_line = line;
            line
        })
        .collect();

    if !is_sorted && !bless {
        tidy_error!(
            bad,
            "`src/tools/tidy/src/issues.txt` is not in order, mostly because you modified it manually,
            please only update it with command `x test tidy --bless`"
        );
    }

    let mut remaining_issue_names: BTreeSet<&str> = allowed_issue_names.clone();

    let (ui, ui_fulldeps) = (path.join("ui"), path.join("ui-fulldeps"));
    let paths = [ui.as_path(), ui_fulldeps.as_path()];
    crate::walk::walk_no_read(&paths, |_, _| false, &mut |entry| {
        let file_path = entry.path();
        if let Some(ext) = file_path.extension().and_then(OsStr::to_str) {
            // files that are neither an expected extension or an exception should not exist
            // they're probably typos or not meant to exist
            if !(EXPECTED_TEST_FILE_EXTENSIONS.contains(&ext)
                || EXTENSION_EXCEPTION_PATHS.iter().any(|path| file_path.ends_with(path)))
            {
                tidy_error!(bad, "file {} has unexpected extension {}", file_path.display(), ext);
            }

            // NB: We do not use file_stem() as some file names have multiple `.`s and we
            // must strip all of them.
            let testname =
                file_path.file_name().unwrap().to_str().unwrap().split_once('.').unwrap().0;
            if ext == "stderr" || ext == "stdout" || ext == "fixed" {
                // Test output filenames have one of the formats:
                // ```
                // $testname.stderr
                // $testname.$mode.stderr
                // $testname.$revision.stderr
                // $testname.$revision.$mode.stderr
                // ```
                //
                // For now, just make sure that there is a corresponding
                // `$testname.rs` file.

                if !file_path.with_file_name(testname).with_extension("rs").exists()
                    && !testname.contains("ignore-tidy")
                {
                    tidy_error!(bad, "Stray file with UI testing output: {:?}", file_path);
                }

                if let Ok(metadata) = fs::metadata(file_path) {
                    if metadata.len() == 0 {
                        tidy_error!(bad, "Empty file with UI testing output: {:?}", file_path);
                    }
                }
            }

            if ext == "rs" {
                if let Some(test_name) = static_regex!(r"^issues?[-_]?(\d{3,})").captures(testname)
                {
                    // these paths are always relative to the passed `path` and always UTF8
                    let stripped_path = file_path
                        .strip_prefix(path)
                        .unwrap()
                        .to_str()
                        .unwrap()
                        .replace(std::path::MAIN_SEPARATOR_STR, "/");

                    if !remaining_issue_names.remove(stripped_path.as_str()) {
                        tidy_error!(
                            bad,
                            "file `tests/{stripped_path}` must begin with a descriptive name, consider `{{reason}}-issue-{issue_n}.rs`",
                            issue_n = &test_name[1],
                        );
                    }
                }
            }
        }
    });

    // if there are any file names remaining, they were moved on the fs.
    // our data must remain up to date, so it must be removed from issues.txt
    // do this automatically on bless, otherwise issue a tidy error
    if bless && (!remaining_issue_names.is_empty() || !is_sorted) {
        let tidy_src = root_path.join("src/tools/tidy/src");
        // instead of overwriting the file, recreate it and use an "atomic rename"
        // so we don't bork things on panic or a contributor using Ctrl+C
        let blessed_issues_path = tidy_src.join("issues_blessed.txt");
        let mut blessed_issues_txt = fs::File::create(&blessed_issues_path).unwrap();
        blessed_issues_txt.write(issues_txt_header.as_bytes()).unwrap();
        // If we changed paths to use the OS separator, reassert Unix chauvinism for blessing.
        for filename in allowed_issue_names.difference(&remaining_issue_names) {
            writeln!(blessed_issues_txt, "{filename}").unwrap();
        }
        let old_issues_path = tidy_src.join("issues.txt");
        fs::rename(blessed_issues_path, old_issues_path).unwrap();
    } else {
        for file_name in remaining_issue_names {
            let mut p = PathBuf::from(path);
            p.push(file_name);
            tidy_error!(
                bad,
                "file `{}` no longer exists and should be removed from the exclusions in `src/tools/tidy/src/issues.txt`",
                p.display()
            );
        }
    }
}