From 87fbd4e5df460c683cef6d772e3d55af09ab8ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 24 Jan 2025 14:16:56 +0100 Subject: [PATCH 1/6] Remove `build-gccjit.sh` script --- .../host-x86_64/x86_64-gnu-llvm-18/Dockerfile | 3 -- .../host-x86_64/x86_64-gnu-llvm-19/Dockerfile | 3 -- .../host-x86_64/x86_64-gnu-tools/Dockerfile | 3 -- src/ci/docker/scripts/build-gccjit.sh | 37 ------------------- 4 files changed, 46 deletions(-) delete mode 100755 src/ci/docker/scripts/build-gccjit.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile index 0a58f337d9ddb..374d12cc087e4 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile @@ -55,9 +55,6 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.thin-lto-import-instr-limit=10 COPY scripts/shared.sh /scripts/ -COPY scripts/build-gccjit.sh /scripts/ - -RUN /scripts/build-gccjit.sh /scripts ARG SCRIPT_ARG diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile index 092847cdfe04c..e0ed2e227f810 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile @@ -55,9 +55,6 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.thin-lto-import-instr-limit=10 COPY scripts/shared.sh /scripts/ -COPY scripts/build-gccjit.sh /scripts/ - -RUN /scripts/build-gccjit.sh /scripts ARG SCRIPT_ARG diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index ab749b3fdd5a6..89806634c6c26 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -90,9 +90,6 @@ ENV HOST_TARGET x86_64-unknown-linux-gnu #ENV FORCE_CI_RUSTC 1 COPY scripts/shared.sh /scripts/ -COPY scripts/build-gccjit.sh /scripts/ - -RUN /scripts/build-gccjit.sh /scripts # For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries # to create a new folder. For reference: diff --git a/src/ci/docker/scripts/build-gccjit.sh b/src/ci/docker/scripts/build-gccjit.sh deleted file mode 100755 index 43ed2270d313f..0000000000000 --- a/src/ci/docker/scripts/build-gccjit.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -GIT_REPO="https://github.com/rust-lang/gcc" - -# This commit hash needs to be updated to use a more recent gcc fork version. -GIT_COMMIT="45648c2edd4ecd862d9f08196d3d6c6ccba79f07" - -set -ex - -cd $1 - -source shared.sh - -# Setting up folders for GCC -curl -L "$GIT_REPO/archive/$GIT_COMMIT.tar.gz" | - tar -xz --transform "s/gcc-$GIT_COMMIT/gcc-src/" - -mkdir gcc-build gcc-install -pushd gcc-build - -# Building GCC. -hide_output \ - ../gcc-src/configure \ - --enable-host-shared \ - --enable-languages=jit \ - --enable-checking=release \ - --disable-bootstrap \ - --disable-multilib \ - --prefix=$(pwd)/../gcc-install \ - -hide_output make -j$(nproc) -hide_output make install - -popd -rm -rf gcc-src gcc-build -ln -s /scripts/gcc-install/lib/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so -ln -s /scripts/gcc-install/lib/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so.0 From a0cf0f2eb42252f58729ce30b13eec9fd0716a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 12 Feb 2025 10:31:55 +0100 Subject: [PATCH 2/6] Update GCC submodule --- src/gcc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gcc b/src/gcc index fd3498bff0b93..e607be166673a 160000 --- a/src/gcc +++ b/src/gcc @@ -1 +1 @@ -Subproject commit fd3498bff0b939dda91d56960acc33d55f2f9cdf +Subproject commit e607be166673a8de9fc07f6f02c60426e556c5f2 From 17472a9f80e6143538f5c29d9bbc0f1acf8dae4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 30 Jan 2025 15:07:24 +0100 Subject: [PATCH 3/6] Improve GCC build in bootstrap --- src/bootstrap/src/core/build_steps/compile.rs | 9 ++++ src/bootstrap/src/core/build_steps/gcc.rs | 51 ++++++++++++++----- src/bootstrap/src/core/build_steps/test.rs | 7 ++- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 479327d63695c..44469da1e648f 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -17,6 +17,7 @@ use std::{env, fs, str}; use serde_derive::Deserialize; +use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags}; use crate::core::build_steps::tool::SourceType; use crate::core::build_steps::{dist, llvm}; use crate::core::builder; @@ -1500,6 +1501,14 @@ impl Step for CodegenBackend { .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + // Ideally, we'd have a separate step for the individual codegen backends, + // like we have in tests (test::CodegenGCC) but that would require a lot of restructuring. + // If the logic gets more complicated, it should probably be done. + if backend == "gcc" { + let gcc = builder.ensure(Gcc { target }); + add_cg_gcc_cargo_flags(&mut cargo, &gcc); + } + let tmp_stamp = BuildStamp::new(&out_dir).with_prefix("tmp"); let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index 98b8635132b0d..3edca8834e063 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -9,13 +9,13 @@ //! ensure that they're always in place if needed. use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::OnceLock; use build_helper::ci::CiEnv; use crate::Kind; -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::builder::{Builder, Cargo, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash}; use crate::utils::exec::command; @@ -29,7 +29,8 @@ pub struct Meta { } pub enum GccBuildStatus { - AlreadyBuilt, + /// libgccjit is already built at this path + AlreadyBuilt(PathBuf), ShouldBuild(Meta), } @@ -41,9 +42,6 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc // Initialize the gcc submodule if not initialized already. builder.config.update_submodule("src/gcc"); - // FIXME (GuillaumeGomez): To be done once gccjit has been built in the CI. - // builder.config.maybe_download_ci_gcc(); - let root = builder.src.join("src/gcc"); let out_dir = builder.gcc_out(target).join("build"); let install_dir = builder.gcc_out(target).join("install"); @@ -70,19 +68,37 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc stamp.path().display() )); } - return GccBuildStatus::AlreadyBuilt; + let path = libgccjit_built_path(&install_dir); + if path.is_file() { + return GccBuildStatus::AlreadyBuilt(path); + } else { + builder.info(&format!( + "GCC stamp is up-to-date, but the libgccjit.so file was not found at `{}`", + path.display(), + )); + } } GccBuildStatus::ShouldBuild(Meta { stamp, out_dir, install_dir, root }) } +/// Returns the path to a libgccjit.so file in the install directory of GCC. +fn libgccjit_built_path(install_dir: &Path) -> PathBuf { + install_dir.join("lib/libgccjit.so") +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Gcc { pub target: TargetSelection, } +#[derive(Clone)] +pub struct GccOutput { + pub libgccjit: PathBuf, +} + impl Step for Gcc { - type Output = bool; + type Output = GccOutput; const ONLY_HOSTS: bool = true; @@ -94,14 +110,14 @@ impl Step for Gcc { run.builder.ensure(Gcc { target: run.target }); } - /// Compile GCC for `target`. - fn run(self, builder: &Builder<'_>) -> bool { + /// Compile GCC (specifically `libgccjit`) for `target`. + fn run(self, builder: &Builder<'_>) -> Self::Output { let target = self.target; // If GCC has already been built, we avoid building it again. let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target) { - GccBuildStatus::AlreadyBuilt => return true, + GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path }, GccBuildStatus::ShouldBuild(m) => m, }; @@ -110,8 +126,9 @@ impl Step for Gcc { let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); + let libgccjit_path = libgccjit_built_path(&install_dir); if builder.config.dry_run() { - return true; + return GccOutput { libgccjit: libgccjit_path }; } // GCC creates files (e.g. symlinks to the downloaded dependencies) @@ -173,11 +190,17 @@ impl Step for Gcc { let lib_alias = install_dir.join("lib/libgccjit.so.0"); if !lib_alias.exists() { - t!(builder.symlink_file(install_dir.join("lib/libgccjit.so"), lib_alias,)); + t!(builder.symlink_file(&libgccjit_path, lib_alias)); } t!(stamp.write()); - true + GccOutput { libgccjit: libgccjit_path } } } + +/// Configures a Cargo invocation so that it can build the GCC codegen backend. +pub fn add_cg_gcc_cargo_flags(cargo: &mut Cargo, gcc: &GccOutput) { + // Add the path to libgccjit.so to the linker search paths. + cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().display())); +} diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 26ed0e5deaa05..c3eceda0d1a6b 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -12,6 +12,7 @@ use clap_complete::shells; use crate::core::build_steps::compile::run_cargo; use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags}; use crate::core::build_steps::llvm::get_llvm_version; use crate::core::build_steps::synthetic_targets::MirOptPanicAbortSyntheticTarget; use crate::core::build_steps::tool::{self, SourceType, Tool}; @@ -3520,6 +3521,8 @@ impl Step for CodegenGCC { let compiler = self.compiler; let target = self.target; + let gcc = builder.ensure(Gcc { target }); + builder.ensure( compile::Std::new(compiler, target) .extra_rust_args(&["-Csymbol-mangling-version=v0", "-Cpanic=abort"]), @@ -3546,6 +3549,7 @@ impl Step for CodegenGCC { .arg("--manifest-path") .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml")); compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + add_cg_gcc_cargo_flags(&mut cargo, &gcc); // Avoid incremental cache issues when changing rustc cargo.env("CARGO_BUILD_INCREMENTAL", "false"); @@ -3578,9 +3582,10 @@ impl Step for CodegenGCC { .env("CG_RUSTFLAGS", "-Alinker-messages") .arg("--") .arg("test") - .arg("--use-system-gcc") .arg("--use-backend") .arg("gcc") + .arg("--gcc-path") + .arg(gcc.libgccjit.parent().unwrap()) .arg("--out-dir") .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc")) .arg("--release") From b4d9d025382c8dd89d5a0f362bde676254d886ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 14 Feb 2025 18:46:52 +0100 Subject: [PATCH 4/6] Install bzip2 It is apparently required to download GCC dependencies. --- src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile index 374d12cc087e4..aefc0f376f689 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ + bzip2 \ g++ \ gcc-multilib \ make \ From 58be00a4f58496fce5df9176270ce63707d948c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 15 Feb 2025 11:54:46 +0100 Subject: [PATCH 5/6] Update license metadata --- license-metadata.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/license-metadata.json b/license-metadata.json index 4cf9bea2861d4..4fb59210854e9 100644 --- a/license-metadata.json +++ b/license-metadata.json @@ -13,7 +13,8 @@ "directories": [], "files": [ "analyzer-decls.h", - "malloc-macro.h" + "malloc-macro.h", + "sarif-path-role.h" ], "license": { "copyright": [ From ba7d5d11ae50554a56a789dbe7a5e8cf08d42c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 17 Feb 2025 12:34:43 +0100 Subject: [PATCH 6/6] Changed `.display()` to `.to_str()` --- src/bootstrap/src/core/build_steps/gcc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index 3edca8834e063..e8b6a577cea5b 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -202,5 +202,5 @@ impl Step for Gcc { /// Configures a Cargo invocation so that it can build the GCC codegen backend. pub fn add_cg_gcc_cargo_flags(cargo: &mut Cargo, gcc: &GccOutput) { // Add the path to libgccjit.so to the linker search paths. - cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().display())); + cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().to_str().unwrap())); }