Rollup merge of #123423 - kjetilkjeka:llvm_bitcode_linker_component, r=Mark-Simulacrum

Distribute LLVM bitcode linker as a preview component
The self-contained LLVM bitcode linker was recently added in #117458. It is currently only in use to link the Nvidia ptx assembly tests when running rustc tests (local or CI). In fact, the linker itself is currently only usable for the `nvptx64-nvidia-cuda` target, but more targets will be supported in the future.
The reason a new linker was needed for the ptx format is that the [old one](https://github.com/denzp/rust-ptx-linker) has not been updated the last few years. It worked fine for a while, but as LLVM changed it broke and the nvptx tests was [disabled in rustc back in 2019](f8f9a28). It was ad-hoc patched and have been used in a sub-optimal state by the community until now.
If this PR is merged, the LLVM bitcode linker will be distributed as a preview component that can be used as a replacement for the old ptx-linker for development in addition to rustc tests. In addition to installing the `llvm-bitcode-linker` component, also the `llvm-tools` component must be installed as the `llvm-bitcode-linker` works by calling llvm tools.
Even though the LLVM bitcode linker is in its early stages it already now provides a lot of value over the old ptx-linker just by working and using up-to-date llvm tooling. By shipping it as a component it will be easier to gather user experience and improving it.
``@petrochenkov`` when installing as a component it will be installed in the self-contained folder and will not work with `-Clink-self-contained=no` (although for some reason I expect to be a bug `-Clink-self-contained=-linker` doesn't properly disable it). However, when building using `x.py build` it will be placed in the `lib/rustlib/<target>/bin` directory and will be available for internal tests even if `-Clink-self-contained=no` is passed.
CC: ``@Mark-Simulacrum`` as I very briefly discussed it with you some months ago https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/.E2.9C.94.20How.20to.20ship.20a.20new.20tool.20.28embedded.20linker.29.20to.20users.3F

Author: jieyouxu

src/bootstrap/src/core/build_steps/dist.rs

@@ -1548,6 +1548,7 @@ impl Step for Extended {
  compiler: builder.compiler(stage, target),
  backend:"cranelift".to_string(),
 });
+add_component!("llvm-bitcode-linker" => LlvmBitcodeLinker{compiler, target});
 
 let etc = builder.src.join("src/etc/installer");
 
@@ -2224,6 +2225,53 @@ impl Step for LlvmTools {
 }
 }
 
+#[derive(Debug,PartialOrd,Ord,Clone,Hash,PartialEq,Eq)]
+pubstructLlvmBitcodeLinker{
+pubcompiler:Compiler,
+pubtarget:TargetSelection,
+}
+
+implStepforLlvmBitcodeLinker{
+typeOutput = Option<GeneratedTarball>;
+constDEFAULT:bool = true;
+constONLY_HOSTS:bool = true;
+
+fnshould_run(run:ShouldRun<'_>) -> ShouldRun<'_>{
+let default = should_build_extended_tool(run.builder,"llvm-bitcode-linker");
+ run.alias("llvm-bitcode-linker").default_condition(default)
+}
+
+fnmake_run(run:RunConfig<'_>){
+ run.builder.ensure(LlvmBitcodeLinker{
+compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+),
+target: run.target,
+});
+}
+
+fnrun(self,builder:&Builder<'_>) -> Option<GeneratedTarball>{
+let compiler = self.compiler;
+let target = self.target;
+
+let llbc_linker =
+ builder.ensure(tool::LlvmBitcodeLinker{ compiler, target,extra_features:vec![]});
+
+let self_contained_bin_dir = format!("lib/rustlib/{}/bin/self-contained", target.triple);
+
+// Prepare the image directory
+letmut tarball = Tarball::new(builder,"llvm-bitcode-linker",&target.triple);
+ tarball.set_overlay(OverlayKind::LlvmBitcodeLinker);
+ tarball.is_preview(true);
+
+ tarball.add_file(llbc_linker, self_contained_bin_dir,0o755);
+
+Some(tarball.generate())
+}
+}
+
 // Tarball intended for internal consumption to ease rustc/std development.
 //
 // Should not be considered stable by end users.

src/bootstrap/src/core/build_steps/install.rs

@@ -300,6 +300,15 @@ install!((self, builder, _config),
 );
 }
 };
+LlvmBitcodeLinker, alias = "llvm-bitcode-linker",Self::should_build(_config), only_hosts:true,{
+ifletSome(tarball) = builder.ensure(dist::LlvmBitcodeLinker{ compiler:self.compiler, target:self.target }){
+ install_sh(builder,"llvm-bitcode-linker",self.compiler.stage,Some(self.target),&tarball);
+} else {
+ builder.info(
+&format!("skipping llvm-bitcode-linker stage{} ({})",self.compiler.stage,self.target),
+);
+}
+};
 );
 
 #[derive(Debug,Clone,Hash,PartialEq,Eq)]

src/bootstrap/src/core/build_steps/tool.rs

@@ -746,9 +746,11 @@ impl Step for LlvmBitcodeLinker {
 .join(exe(bin_name,self.compiler.host));
 
 ifself.compiler.stage > 0{
-let bindir = builder.sysroot(self.compiler).join("bin");
-t!(fs::create_dir_all(&bindir));
-let bin_destination = bindir.join(exe(bin_name,self.compiler.host));
+let bindir_self_contained = builder
+.sysroot(self.compiler)
+.join(format!("lib/rustlib/{}/bin/self-contained",self.target.triple));
+t!(fs::create_dir_all(&bindir_self_contained));
+let bin_destination = bindir_self_contained.join(exe(bin_name,self.compiler.host));
  builder.copy_link(&tool_out,&bin_destination);
  bin_destination
 }else{

src/bootstrap/src/core/builder.rs

@@ -853,6 +853,7 @@ impl<'a> Builder<'a> {
  dist::Clippy,
  dist::Miri,
  dist::LlvmTools,
+ dist::LlvmBitcodeLinker,
  dist::RustDev,
  dist::Bootstrap,
  dist::Extended,

src/bootstrap/src/utils/tarball.rs

@@ -20,6 +20,7 @@ pub(crate) enum OverlayKind {
 RLS,
 RustAnalyzer,
 RustcCodegenCranelift,
+LlvmBitcodeLinker,
 }
 
 implOverlayKind{
@@ -64,6 +65,12 @@ impl OverlayKind {
 "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
 "compiler/rustc_codegen_cranelift/LICENSE-MIT",
 ],
+OverlayKind::LlvmBitcodeLinker => &[
+"COPYRIGHT",
+"LICENSE-APACHE",
+"LICENSE-MIT",
+"src/tools/llvm-bitcode-linker/README.md",
+],
 }
 }
 
@@ -87,6 +94,7 @@ impl OverlayKind {
 .rust_analyzer_info
 .version(builder,&builder.release_num("rust-analyzer/crates/rust-analyzer")),
 OverlayKind::RustcCodegenCranelift => builder.rust_version(),
+OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
 }
 }
 }

src/doc/unstable-book/src/compiler-flags/codegen-options.md

@@ -10,6 +10,10 @@ In addition to the stable set of linker flavors, the following unstable values a
 -`ptx`: use [`rust-ptx-linker`](https://github.com/denzp/rust-ptx-linker)
  for Nvidia NVPTX GPGPU support.
 -`bpf`: use [`bpf-linker`](https://github.com/alessandrod/bpf-linker) for eBPF support.
+-`llbc`: for linking in llvm bitcode. Install the preview rustup components`llvm-bitcode-linker`
+ and `llvm-tools` to use as a self-contained linker by passing
+`-Zunstable-options -Clink-self-contained=+linker` together with `-Clinker-flavor=llbc`.
+ Can currently only be used for Nvidia NVPTX targets (`nvptx64-nvidia-cuda`).
 
 Additionally, a set of more precise linker flavors also exists, for example allowing targets to
 declare that they use the LLD linker by default. The following values are currently unstable, and

src/tools/llvm-bitcode-linker/src/linker.rs

@@ -62,7 +62,7 @@ impl Session {
 .arg("-o")
 .arg(&self.link_path)
 .output()
-.unwrap();
+.context("An error occured when calling llvm-link. Make sure the llvm-tools component is installed.")?;
 
 if !llvm_link_output.status.success(){
  tracing::error!(
@@ -108,7 +108,9 @@ impl Session {
  opt_cmd.arg("--strip-debug");
 }
 
-let opt_output = opt_cmd.output().unwrap();
+let opt_output = opt_cmd.output().context(
+"An error occured when calling opt. Make sure the llvm-tools component is installed.",
+)?;
 
 if !opt_output.status.success(){
  tracing::error!(
@@ -133,8 +135,11 @@ impl Session {
  lcc_command.arg("--mcpu").arg(mcpu);
 }
 
-let lcc_output =
- lcc_command.arg(&self.opt_path).arg("-o").arg(&self.out_path).output().unwrap();
+let lcc_output = lcc_command
+.arg(&self.opt_path)
+.arg("-o").arg(&self.out_path)
+.output()
+.context("An error occured when calling llc. Make sure the llvm-tools component is installed.")?;
 
 if !lcc_output.status.success(){
  tracing::error!(