rustc_codegen_ssa: Correctly apply the static --jobs-backend limit to backend parallelism

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -1,4 +1,5 @@
 use std::marker::PhantomData;
+use std::num::NonZero;
 use std::panic::AssertUnwindSafe;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -356,10 +357,10 @@ pub struct CodegenContext {
     /// The incremental compilation session directory, or None if we are not
     /// compiling incrementally
     pub incr_comp_session_dir: Option<PathBuf>,
-    /// `true` if the codegen should be run in parallel.
+    /// `Some(limit)` if the codegen should be run in parallel.
     ///
     /// Depends on [`WriteBackendMethods::supports_parallel()`] and `--jobs-backend`.
-    pub parallel: bool,
+    pub parallel: Option<NonZero<usize>>,
 }
 
 fn generate_thin_lto_work<B: WriteBackendMethods>(
@@ -1021,7 +1022,7 @@ fn do_thin_lto<B: WriteBackendMethods>(
     // Note that using `jobserver::Proxy` is not necessary here, the code below always acquires
     // tokens before releasing them, so we can never accidentally release the last token
     // permanently held by rustc process.
-    let jobserver_helper = cgcx.parallel.then(|| {
+    let jobserver_helper = cgcx.parallel.map(|_| {
         let coordinator_send2 = coordinator_send.clone();
         jobserver::client()
             .into_helper_thread(move |token| {
@@ -1037,18 +1038,23 @@ fn do_thin_lto<B: WriteBackendMethods>(
     // bunch of work items onto our queue to do LTO. This all
     // happens on the coordinator thread but it's very quick so
     // we don't worry about tokens.
-    for (work, cost) in generate_thin_lto_work::<B>(
+    for (i, (work, cost)) in generate_thin_lto_work::<B>(
         cgcx,
         prof,
         dcx,
         &exported_symbols_for_lto,
         &each_linked_rlib_for_lto,
         needs_thin_lto,
-    ) {
+    )
+    .into_iter()
+    .enumerate()
+    {
         let insertion_index =
             work_items.binary_search_by_key(&cost, |&(_, cost)| cost).unwrap_or_else(|e| e);
         work_items.insert(insertion_index, (work, cost));
-        if let Some(helper) = &jobserver_helper {
+        if let Some(helper) = &jobserver_helper
+            && i < cgcx.parallel.unwrap().get()
+        {
             helper.request_token();
         }
     }
@@ -1255,12 +1261,11 @@ fn start_executing_work<B: WriteBackendMethods>(
     // Note that using `jobserver::Proxy` is not necessary here, the code below always acquires
     // tokens before releasing them, so we can never accidentally release the last token
     // permanently held by rustc process.
-    // FIXME: the backend parallelism is currently limited solely by the jobserver,
-    // so if `--jobs-backend` is smaller than `--jobs(-frontend)`, or than the number of tokens
-    // that the external jobserver can give, then it won't be respected.
-    // Below we'll need to add some additional work limiting for `--jobs-backend` to be respected.
-    let parallel = sess.opts.jobs.backend.is_some() && backend.supports_parallel();
-    let jobserver_helper = parallel.then(|| {
+    let parallel = match sess.opts.jobs.backend {
+        Some(n) if backend.supports_parallel() => Some(n),
+        _ => None,
+    };
+    let jobserver_helper = parallel.map(|_| {
         let coordinator_send2 = coordinator_send.clone();
         jobserver::client()
             .into_helper_thread(move |token| {
@@ -1655,7 +1660,10 @@ fn start_executing_work<B: WriteBackendMethods>(
                     };
                     work_items.insert(insertion_index, (llvm_work_item, cost));
 
-                    if let Some(helper) = &jobserver_helper {
+                    if let Some(helper) = &jobserver_helper
+                        && running_with_any_token(main_thread_state, running_with_own_token)
+                            < cgcx.parallel.unwrap().get()
+                    {
                         helper.request_token();
                     }
                     assert_eq!(main_thread_state, MainThreadState::Codegenning);

compiler/rustc_interface/src/interface.rs

@@ -375,7 +375,8 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
 
     // Initialize jobserver as early as possible.
     let early_dcx = EarlyDiagCtxt::new(config.opts.error_format);
-    if let Some(limit) = config.opts.jobs.frontend.max(config.opts.jobs.backend) {
+    let jobs = config.opts.jobs;
+    if let Some(limit) = jobs.frontend.max(jobs.backend).max(jobs.linker.limit()) {
         jobserver::initialize(limit.get(), |err| {
             let note = "the build environment is likely misconfigured";
             early_dcx.early_struct_warn(err).with_note(note).emit()
@@ -398,7 +399,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
     util::run_in_thread_pool_with_globals(
         &early_dcx,
         config.opts.edition,
-        config.opts.jobs,
+        jobs,
         &config.extra_symbols,
         SourceMapInputs { file_loader, path_mapping, hash_kind, checksum_hash_kind },
         |current_gcx| {

compiler/rustc_session/src/config.rs

@@ -1665,6 +1665,15 @@ pub enum LinkerJobs {
     Explicit(NonZero<usize>),
 }
 
+impl LinkerJobs {
+    pub fn limit(self) -> Option<NonZero<usize>> {
+        match self {
+            LinkerJobs::Default => None,
+            LinkerJobs::Explicit(n) => Some(n),
+        }
+    }
+}
+
 /// `None` for frontend and backend means everything is single-threaded
 /// and synchronization can be disabled.
 #[derive(Clone, Copy)]

src/doc/rustc/src/command-line-arguments.md

@@ -516,10 +516,6 @@ Parallelism used by compilation stages converting backend IR to object files.
 In any case the parallelism here may be additionally limited dynamically by jobserver
 passed from a higher level build system like cargo.
 
-Note: the backend parallelism limit may currently work incorrectly if `jobs-frontend` or `jobs`
-have larger value than `jobs-backend`, or if the inherited jobserver can give a larger number
-of tokens.
-
 ### Linker parallelism
 
 Parallelism used by linker when combining object files into a final binary.

tests/ui/compile-flags/jobs/jobs-pass-link.rs

@@ -1,4 +1,4 @@
 //@ build-pass
-//@ compile-flags: -Z unstable-options --jobs-linker 2
+//@ compile-flags: -Z unstable-options --jobs-linker 2 --jobs-backend 1
 
 fn main() {}