Auto merge of #122997 - matthiaskrgr:compiletest_ices, r=oli-obk

compiletest ice tracking
see https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/where.20to.20mass-add.20known.20ices.20.2F.20merging.20glacier.20into.20rust/near/429082963
This will allow us to sunset most of https://github.com/rust-lang/glacier
The rustc ices will be tracked directly inside the rust testsuite
There are a couple of .sh tests remaining that I have not ported over yet.
This adds `tests/crashes`, a file inside this directory MUST ice, otherwise it is considered test-fail.
This will be used to track ICEs from glacier and the bugtracker.
When someones pr accidentally fixes one of these ICEs, they can move the test from `crashes` into `ui` for example.
I also added a new tidy lint that warns when a test inside `tests/crashes` does not have a `//@ known-bug: ` line
the env var `COMPILETEST_VERBOSE_CRASHES` can be set to get exit code, stderr and stdout of a crash-test to aid debugging/adding tests.

Author: bors

src/bootstrap/src/core/build_steps/test.rs

@@ -1400,6 +1400,8 @@ impl Step for RunMakeSupport {
 
 default_test!(Ui{ path:"tests/ui", mode:"ui", suite:"ui"});
 
+default_test!(Crashes{ path:"tests/crashes", mode:"crashes", suite:"crashes"});
+
 default_test!(RunPassValgrind{
  path:"tests/run-pass-valgrind",
  mode:"run-pass-valgrind",

src/bootstrap/src/core/builder.rs

@@ -745,6 +745,7 @@ impl<'a> Builder<'a> {
  test::ExpandYamlAnchors,
  test::Tidy,
  test::Ui,
+ test::Crashes,
  test::RunPassValgrind,
  test::Coverage,
  test::CoverageMap,

src/tools/compiletest/src/common.rs

@@ -69,6 +69,7 @@ string_enum! {
 Assembly => "assembly",
 CoverageMap => "coverage-map",
 CoverageRun => "coverage-run",
+Crashes => "crashes",
 }
 }
 

src/tools/compiletest/src/header.rs

@@ -581,6 +581,16 @@ impl TestProps {
 self.incremental = true;
 }
 
+if config.mode == Mode::Crashes{
+// we don't want to pollute anything with backtrace-files
+// also turn off backtraces in order to save some execution
+// time on the tests; we only need to know IF it crashes
+self.rustc_env = vec![
+("RUST_BACKTRACE".to_string(),"0".to_string()),
+("RUSTC_ICE".to_string(),"0".to_string()),
+];
+}
+
 for key in&["RUST_TEST_NOCAPTURE","RUST_TEST_THREADS"]{
 ifletOk(val) = env::var(key){
 ifself.exec_env.iter().find(|&&(ref x, _)| x == key).is_none(){
@@ -596,7 +606,8 @@ impl TestProps {
 
 fnupdate_fail_mode(&mutself,ln:&str,config:&Config){
 let check_ui = |mode:&str| {
-if config.mode != Mode::Ui{
+// Mode::Crashes may need build-fail in order to trigger llvm errors or stack overflows
+if config.mode != Mode::Ui && config.mode != Mode::Crashes{
 panic!("`{}-fail` header is only supported in UI tests", mode);
 }
 };
@@ -625,6 +636,7 @@ impl TestProps {
 fnupdate_pass_mode(&mutself,ln:&str,revision:Option<&str>,config:&Config){
 let check_no_run = |s| match(config.mode, s){
 (Mode::Ui, _) => (),
+(Mode::Crashes, _) => (),
 (Mode::Codegen,"build-pass") => (),
 (Mode::Incremental, _) => {
 if revision.is_some() && !self.revisions.iter().all(|r| r.starts_with("cfail")){
@@ -757,6 +769,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
 "ignore-mode-codegen-units",
 "ignore-mode-coverage-map",
 "ignore-mode-coverage-run",
+"ignore-mode-crashes",
 "ignore-mode-debuginfo",
 "ignore-mode-incremental",
 "ignore-mode-js-doc-test",

src/tools/compiletest/src/lib.rs

@@ -65,7 +65,8 @@ pub fn parse_config(args: Vec<String>) -> Config {
 "mode",
 "which sort of compile tests to run",
 "run-pass-valgrind | pretty | debug-info | codegen | rustdoc \
- | rustdoc-json | codegen-units | incremental | run-make | ui | js-doc-test | mir-opt | assembly",
+ | rustdoc-json | codegen-units | incremental | run-make | ui \
+ | js-doc-test | mir-opt | assembly | crashes",
 )
 .reqopt(
 "",
@@ -82,7 +83,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
 .optopt("","run","whether to execute run-* tests","auto | always | never")
 .optflag("","ignored","run tests marked as ignored")
 .optflag("","with-debug-assertions","whether to run tests with `ignore-debug` header")
-.optmulti("","skip","skip tests matching SUBSTRING. Can be passed multiple times","SUBSTRING")
+.optmulti(
+"",
+"skip",
+"skip tests matching SUBSTRING. Can be passed multiple times",
+"SUBSTRING",
+)
 .optflag("","exact","filters match exactly")
 .optopt(
 "",
@@ -145,7 +151,11 @@ pub fn parse_config(args: Vec<String>) -> Config {
 .optflag("","profiler-support","is the profiler runtime enabled for this target")
 .optflag("h","help","show this message")
 .reqopt("","channel","current Rust channel","CHANNEL")
-.optflag("","git-hash","run tests which rely on commit version being compiled into the binaries")
+.optflag(
+"",
+"git-hash",
+"run tests which rely on commit version being compiled into the binaries",
+)
 .optopt("","edition","default Rust edition","EDITION")
 .reqopt("","git-repository","name of the git repository","ORG/REPO")
 .reqopt("","nightly-branch","name of the git branch for nightly","BRANCH");

src/tools/compiletest/src/runtest.rs

@@ -4,7 +4,7 @@ use crate::common::{
  expected_output_path,UI_EXTENSIONS,UI_FIXED,UI_STDERR,UI_STDOUT,UI_SVG,UI_WINDOWS_SVG,
 };
 usecrate::common::{incremental_dir, output_base_dir, output_base_name, output_testname_unique};
-usecrate::common::{Assembly,Incremental,JsDocTest,MirOpt,RunMake,RustdocJson,Ui};
+usecrate::common::{Assembly,Crashes,Incremental,JsDocTest,MirOpt,RunMake,RustdocJson,Ui};
 usecrate::common::{Codegen,CodegenUnits,DebugInfo,Debugger,Rustdoc};
 usecrate::common::{CompareMode,FailMode,PassMode};
 usecrate::common::{Config,TestPaths};
@@ -244,7 +244,7 @@ impl<'test> TestCx<'test> {
 /// Code executed for each revision in turn (or, if there are no
 /// revisions, exactly once, with revision == None).
 fnrun_revision(&self){
-ifself.props.should_ice && self.config.mode != Incremental{
+ifself.props.should_ice && self.config.mode != Incremental&& self.config.mode != Crashes{
 self.fatal("cannot use should-ice in a test that is not cfail");
 }
 matchself.config.mode{
@@ -263,6 +263,7 @@ impl<'test> TestCx<'test> {
 JsDocTest => self.run_js_doc_test(),
 CoverageMap => self.run_coverage_map_test(),
 CoverageRun => self.run_coverage_run_test(),
+Crashes => self.run_crash_test(),
 }
 }
 
@@ -295,6 +296,7 @@ impl<'test> TestCx<'test> {
 matchself.config.mode{
 JsDocTest => true,
 Ui => pm.is_some() || self.props.fail_mode > Some(FailMode::Build),
+Crashes => false,
 Incremental => {
 let revision =
 self.revision.expect("incremental tests require a list of revisions");
@@ -359,6 +361,27 @@ impl<'test> TestCx<'test> {
 self.check_forbid_output(&output_to_check,&proc_res);
 }
 
+fnrun_crash_test(&self){
+let pm = self.pass_mode();
+let proc_res = self.compile_test(WillExecute::No,self.should_emit_metadata(pm));
+
+if std::env::var("COMPILETEST_VERBOSE_CRASHES").is_ok(){
+eprintln!("{}", proc_res.status);
+eprintln!("{}", proc_res.stdout);
+eprintln!("{}", proc_res.stderr);
+eprintln!("{}", proc_res.cmdline);
+}
+
+// if a test does not crash, consider it an error
+if proc_res.status.success() || matches!(proc_res.status.code(),Some(1 | 0)){
+self.fatal(&format!(
+"test no longer crashes/triggers ICE! Please give it a mearningful name, \
+ add a doc-comment to the start of the test explaining why it exists and \
+ move it to tests/ui or wherever you see fit."
+));
+}
+}
+
 fnrun_rfail_test(&self){
 let pm = self.pass_mode();
 let should_run = self.run_if_enabled();
@@ -2517,7 +2540,7 @@ impl<'test> TestCx<'test> {
  rustc.arg("-Cdebug-assertions=no");
 }
 RunPassValgrind | Pretty | DebugInfo | Rustdoc | RustdocJson | RunMake
- | CodegenUnits | JsDocTest => {
+ | CodegenUnits | JsDocTest| Crashes=> {
 // do not use JSON output
 }
 }

src/tools/opt-dist/src/tests.rs

@@ -96,6 +96,7 @@ llvm-config = "{llvm_config}"
 "tests/pretty",
 "tests/run-pass-valgrind",
 "tests/ui",
+"tests/crashes",
 ];
 for test_path in env.skipped_tests(){
  args.extend(["--skip", test_path]);

src/tools/tidy/src/known_bug.rs

@@ -0,0 +1,17 @@
+//! Tidy check to ensure that tests inside 'tests/crashes' have a '@known-bug' directive.
+
+usecrate::walk::*;
+use std::path::Path;
+
+pubfncheck(filepath:&Path,bad:&mutbool){
+walk(filepath, |path, _is_dir| filter_not_rust(path),&mut |entry, contents| {
+let file = entry.path();
+if !contents.lines().any(|line| line.starts_with("//@ known-bug: ")){
+tidy_error!(
+ bad,
+"{} crash/ice test does not have a \"//@ known-bug: \" directive",
+ file.display()
+);
+}
+});
+}

src/tools/tidy/src/lib.rs

@@ -67,6 +67,7 @@ pub mod features;
 pubmod fluent_alphabetical;
 mod fluent_used;
 pub(crate)mod iter_header;
+pubmod known_bug;
 pubmod mir_opt_tests;
 pubmod pal;
 pubmod run_make_tests;

src/tools/tidy/src/main.rs

@@ -35,6 +35,7 @@ fn main() {
 let library_path = root_path.join("library");
 let compiler_path = root_path.join("compiler");
 let librustdoc_path = src_path.join("librustdoc");
+let crashes_path = tests_path.join("crashes");
 
 let args:Vec<String> = env::args().skip(1).collect();
 let(cfg_args, pos_args) = match args.iter().position(|arg| arg == "--"){
@@ -108,6 +109,7 @@ fn main() {
 check!(mir_opt_tests,&tests_path, bless);
 check!(rustdoc_gui_tests,&tests_path);
 check!(rustdoc_css_themes,&librustdoc_path);
+check!(known_bug,&crashes_path);
 
 // Checks that only make sense for the compiler.
 check!(error_codes,&root_path,&[&compiler_path,&librustdoc_path], verbose);