diff --git a/Cargo.lock b/Cargo.lock index ce3fd04faf3bb..3ddb0c6981318 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4528,12 +4528,14 @@ name = "rustc_monomorphize" version = "0.0.0" dependencies = [ "rustc_abi", + "rustc_ast", "rustc_data_structures", "rustc_errors", "rustc_hir", "rustc_index", "rustc_macros", "rustc_middle", + "rustc_serialize", "rustc_session", "rustc_span", "rustc_symbol_mangling", diff --git a/compiler/rustc_builtin_macros/src/offload.rs b/compiler/rustc_builtin_macros/src/offload.rs index e47ccc0d85f7d..006332b8c40c9 100644 --- a/compiler/rustc_builtin_macros/src/offload.rs +++ b/compiler/rustc_builtin_macros/src/offload.rs @@ -1,6 +1,6 @@ +use rustc_ast::ast; use rustc_ast::token::{Delimiter, Token, TokenKind}; use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree}; -use rustc_ast::{AttrItem, ast}; use rustc_expand::base::{Annotatable, ExtCtxt}; use rustc_session::config::Offload; use rustc_span::{DUMMY_SP, Ident, Span, sym}; @@ -9,7 +9,7 @@ use thin_vec::thin_vec; use crate::diagnostics; fn compile_for_device(ecx: &mut ExtCtxt<'_>) -> bool { - ecx.sess.opts.unstable_opts.offload.contains(&Offload::Device) + ecx.sess.opts.unstable_opts.offload.iter().any(|o| matches!(o, Offload::Device(_))) } fn outer_normal_attr(normal: &Box, id: ast::AttrId, span: Span) -> ast::Attribute { @@ -45,7 +45,6 @@ fn extract_fn( /// This expands to the host-side function: /// /// ``` -/// #[unsafe(no_mangle)] /// #[inline(never)] /// fn foo(_: &[f32], _: &[f32], _: *mut f32) { /// ::core::panicking::panic("not implemented") @@ -56,7 +55,6 @@ fn extract_fn( /// /// ``` /// #[rustc_offload_kernel] -/// #[unsafe(no_mangle)] /// unsafe extern "gpu-kernel" fn foo(a: &[f32], b: &[f32], c: *mut f32) { /// *c = a[0] + b[0]; /// } @@ -110,24 +108,9 @@ pub(crate) fn expand_kernel( span, ); - // unsafe(no_mangle) attr - let unsafe_item = AttrItem { - unsafety: ast::Safety::Unsafe(span), - path: ast::Path::from_ident(Ident::new(sym::no_mangle, span)), - args: ast::AttrArgs::Empty, - span, - }; - - let no_mangle_attr = Box::new(ast::NormalAttr { item: unsafe_item, tokens: None }); - let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); - let unsafe_no_mangle = outer_normal_attr(&no_mangle_attr, new_id, span); - let device_item = { - let mut item = ecx.item( - span, - thin_vec![rustc_offload_kernel, unsafe_no_mangle], - ast::ItemKind::Fn(device_fn), - ); + let mut item = + ecx.item(span, thin_vec![rustc_offload_kernel.clone()], ast::ItemKind::Fn(device_fn)); item.vis = vis.clone(); Annotatable::Item(item) }; @@ -187,12 +170,12 @@ pub(crate) fn expand_kernel( let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let inline_never = outer_normal_attr(&inline_never_attr, new_id, span); - let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); - let unsafe_no_mangle = outer_normal_attr(&no_mangle_attr, new_id, span); - let host_item = { - let mut item = - ecx.item(span, thin_vec![unsafe_no_mangle, inline_never], ast::ItemKind::Fn(host_fn)); + let mut item = ecx.item( + span, + thin_vec![rustc_offload_kernel, inline_never], + ast::ItemKind::Fn(host_fn), + ); item.vis = vis.clone(); Annotatable::Item(item) }; diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 400a9ce89df8b..66b51d9184a79 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -742,7 +742,9 @@ pub(crate) unsafe fn llvm_optimize( llvm::set_value_name(new_fn, &name); } - if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Device) { + if cgcx.target_is_like_gpu + && config.offload.iter().any(|o| matches!(o, config::Offload::Device(_))) + { let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); for func in cx.get_functions() { @@ -813,7 +815,9 @@ pub(crate) unsafe fn llvm_optimize( ) }; - if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Device) { + if cgcx.target_is_like_gpu + && config.offload.iter().any(|o| matches!(o, config::Offload::Device(_))) + { let device_path = cgcx.output_filenames.path(OutputType::Object); let device_dir = device_path.parent().unwrap(); let device_out = device_dir.join("device.bin"); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4ae85af897527..ba11ef29fb536 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -26,7 +26,9 @@ use rustc_session::config::CrateType; use rustc_session::diagnostics::feature_err; use rustc_session::lint::builtin::DEPRECATED_LLVM_INTRINSIC; use rustc_span::{ErrorGuaranteed, Span, Symbol, sym}; -use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; +use rustc_symbol_mangling::{ + mangle_internal_symbol, mangle_offload_export, symbol_name_for_instance_in_crate, +}; use rustc_target::callconv::PassMode; use rustc_target::spec::Arch; use tracing::debug; @@ -1850,9 +1852,9 @@ fn codegen_offload<'ll, 'tcx>( _ => panic!("unparsable"), }; let args = get_args_from_tuple(bx, args[4], fn_target); - let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE); + let target_symbol = mangle_offload_export(tcx, fn_target); - let sig = tcx.fn_sig(fn_target.def_id()).skip_binder(); + let sig = tcx.fn_sig(fn_target.def_id()).instantiate(tcx, fn_target.args).skip_norm_wip(); let sig = tcx.instantiate_bound_regions_with_erased(sig); let inputs = sig.inputs(); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 1f0e583709592..552a91ffee071 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -375,7 +375,7 @@ impl CodegenBackend for LlvmCodegenBackend { fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Box { use rustc_session::config::Offload; - if tcx.sess.opts.unstable_opts.offload.contains(&Offload::Device) + if tcx.sess.opts.unstable_opts.offload.iter().any(|o| matches!(o, Offload::Device(_))) || tcx.sess.opts.unstable_opts.offload.iter().any(|o| matches!(o, Offload::Host(_))) { match llvm::RustOffloadWrapper::get_or_init(&tcx.sess.opts.sysroot) { diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 941e9d28fc7e1..cc102c636f4ae 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -3,6 +3,7 @@ use std::collections::hash_map::Entry::*; use rustc_abi::{CanonAbi, X86Call}; use rustc_ast::expand::allocator::{AllocatorKind, NO_ALLOC_SHIM_IS_UNSTABLE, global_fn_name}; use rustc_crate_store::CrateDepKind; +use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; use rustc_hir::def::DefKind; @@ -19,7 +20,7 @@ use rustc_middle::ty::{ use rustc_middle::util::Providers; use rustc_session::config::CrateType; use rustc_span::Span; -use rustc_symbol_mangling::mangle_internal_symbol; +use rustc_symbol_mangling::{is_offload_kernel, mangle_internal_symbol}; use rustc_target::spec::{Arch, Os, TlsModel}; use tracing::debug; @@ -244,6 +245,51 @@ pub fn exported_non_generic_symbols_helper<'tcx>( )); } + let is_device_offload = tcx + .sess + .opts + .unstable_opts + .offload + .iter() + .any(|o| matches!(o, rustc_session::config::Offload::Device(_))); + if is_device_offload { + let crate_items = tcx.hir_crate_items(()); + let mut seen: rustc_data_structures::fx::FxHashSet = symbols + .iter() + .filter_map(|(s, _)| match s { + ExportedSymbol::NonGeneric(d) => Some(*d), + _ => None, + }) + .collect(); + + let mut try_emit_offload_kernel = |def_id: DefId, seen: &mut FxHashSet| { + if !matches!(tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) { + return; + } + if !tcx.generics_of(def_id).requires_monomorphization(tcx) + && is_offload_kernel(tcx.codegen_fn_attrs(def_id)) + && seen.insert(def_id) + { + symbols.push(( + ExportedSymbol::NonGeneric(def_id), + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Text, + used: false, + rustc_std_internal_symbol: false, + }, + )); + } + }; + + for id in crate_items.free_items() { + try_emit_offload_kernel(id.owner_id.to_def_id(), &mut seen); + } + for id in crate_items.impl_items() { + try_emit_offload_kernel(id.owner_id.to_def_id(), &mut seen); + } + } + // Sort so we get a stable incr. comp. hash. symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx)); @@ -260,7 +306,16 @@ fn exported_generic_symbols_provider_local<'tcx>( let mut symbols: Vec<_> = vec![]; - if tcx.local_crate_exports_generics() { + let export_generics = tcx.local_crate_exports_generics(); + let is_device_offload = tcx + .sess + .opts + .unstable_opts + .offload + .iter() + .any(|o| matches!(o, rustc_session::config::Offload::Device(_))); + + if export_generics || is_device_offload { use rustc_hir::attrs::Linkage; use rustc_middle::mono::{MonoItem, Visibility}; use rustc_middle::ty::InstanceKind; @@ -306,6 +361,14 @@ fn exported_generic_symbols_provider_local<'tcx>( }) }; + let is_offload_instance = |mono_item: &MonoItem<'tcx>| { + if let MonoItem::Fn(instance) = mono_item { + is_offload_kernel(tcx.codegen_fn_attrs(instance.def_id())) + } else { + false + } + }; + // The symbols created in this loop are sorted below it #[allow(rustc::potential_query_instability)] for (mono_item, data) in cgus.iter().flat_map(|cgu| cgu.items().iter()) { @@ -321,7 +384,9 @@ fn exported_generic_symbols_provider_local<'tcx>( continue; } - if !tcx.sess.opts.share_generics() { + let item_is_offload = is_offload_instance(mono_item); + + if !item_is_offload && !tcx.sess.opts.share_generics() { if tcx.codegen_fn_attrs(mono_item.def_id()).inline == rustc_hir::attrs::InlineAttr::Never { @@ -338,15 +403,22 @@ fn exported_generic_symbols_provider_local<'tcx>( MonoItem::Fn(Instance { def: InstanceKind::Item(def), args }) => { let has_generics = args.non_erasable_generics().next().is_some(); - let should_export = - has_generics && is_instantiable_downstream(Some(def), &args); + let should_export = if item_is_offload { + has_generics + } else { + has_generics && is_instantiable_downstream(Some(def), &args) + }; if should_export { let symbol = ExportedSymbol::Generic(def, args); symbols.push(( symbol, SymbolExportInfo { - level: SymbolExportLevel::Rust, + level: if item_is_offload { + SymbolExportLevel::C + } else { + SymbolExportLevel::Rust + }, kind: SymbolExportKind::Text, used: false, rustc_std_internal_symbol: false, diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 0e5bf519fab18..d8e5243148847 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -1313,11 +1313,25 @@ pub(crate) fn start_codegen<'tcx>( let metadata = rustc_metadata::fs::encode_and_write_metadata(tcx); + let is_host_metadata = tcx + .sess + .opts + .unstable_opts + .offload + .iter() + .any(|o| matches!(o, rustc_session::config::Offload::HostMetadata(_))); + let codegen = tcx.sess.time("codegen_crate", || { - if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - // Skip crate items and just output metadata in -Z no-codegen mode. + if tcx.sess.opts.unstable_opts.no_codegen + || !tcx.sess.opts.output_types.should_codegen() + || is_host_metadata + { tcx.sess.dcx().abort_if_errors(); + if is_host_metadata { + rustc_monomorphize::write_host_metadata_offload_manifest(tcx); + } + // Linker::link will skip join_codegen in case of a CodegenResults Any value. Box::new(CompiledModules { modules: vec![], allocator_module: None }) } else { diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 548ee3f4b8e7b..24c7ff8484a5e 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -857,7 +857,7 @@ fn test_unstable_options_tracking_hash() { tracked!(no_profiler_runtime, true); tracked!(no_trait_vptr, true); tracked!(no_unique_section_names, true); - tracked!(offload, vec![Offload::Device]); + tracked!(offload, vec![Offload::Device(String::new())]); tracked!(on_broken_pipe, OnBrokenPipe::Kill); tracked!(osx_rpath_install_name, true); tracked!(packed_bundled_libs, true); diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs index b6ae4a98a34e3..ef9043101dbbb 100644 --- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs +++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs @@ -295,6 +295,8 @@ impl CodegenFnAttrs { // note: for these we do also set a symbol name so technically also handled by the // condition below. However, I think that regardless these should be treated as extern. || self.flags.contains(CodegenFnAttrFlags::EXTERNALLY_IMPLEMENTABLE_ITEM) + // `#[rustc_offload_kernel]`: this item is an externally-launched kernel entry point. + || self.flags.contains(CodegenFnAttrFlags::OFFLOAD_KERNEL) || self.symbol_name.is_some() || match self.linkage { // These are private, so make sure we don't try to consider diff --git a/compiler/rustc_monomorphize/Cargo.toml b/compiler/rustc_monomorphize/Cargo.toml index 58ccf77903bab..8846de7a9bf81 100644 --- a/compiler/rustc_monomorphize/Cargo.toml +++ b/compiler/rustc_monomorphize/Cargo.toml @@ -6,12 +6,14 @@ edition = "2024" [dependencies] # tidy-alphabetical-start rustc_abi = { path = "../rustc_abi" } +rustc_ast = { path = "../rustc_ast" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_hir = { path = "../rustc_hir" } rustc_index = { path = "../rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_middle = { path = "../rustc_middle" } +rustc_serialize = { path = "../rustc_serialize" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } @@ -20,3 +22,8 @@ serde = "1" serde_json = "1" tracing = "0.1" # tidy-alphabetical-end + +[features] +# tidy-alphabetical-start +llvm_offload = [] +# tidy-alphabetical-end diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 1ee6e0506dcdd..5d7820df622b7 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -231,8 +231,8 @@ use rustc_middle::ty::{ }; use rustc_middle::util::Providers; use rustc_middle::{bug, span_bug}; -use rustc_session::config::{DebugInfo, EntryFnType}; -use rustc_span::{DUMMY_SP, Span, Spanned, dummy_spanned, respan}; +use rustc_session::config::{DebugInfo, EntryFnType, Offload}; +use rustc_span::{DUMMY_SP, Span, Spanned, Symbol, dummy_spanned, respan}; use tracing::{debug, instrument, trace}; use crate::diagnostics::{ @@ -831,8 +831,8 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> { }; match terminator.kind { - mir::TerminatorKind::Call { ref func, .. } - | mir::TerminatorKind::TailCall { ref func, .. } => { + mir::TerminatorKind::Call { ref func, ref args, .. } + | mir::TerminatorKind::TailCall { ref func, ref args, .. } => { let callee_ty = func.ty(self.body, tcx); // *Before* monomorphizing, record that we already handled this mention. self.used_mentioned_items.insert(MentionedItem::Fn(callee_ty)); @@ -865,7 +865,16 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> { !force_indirect_call, source, &mut self.used_items, - ) + ); + + if let ty::FnDef(def_id, _) = *callee_ty.kind() + && self.tcx.is_intrinsic(def_id, rustc_span::sym::offload) + && let Some(kernel) = args.first() + { + let kernel_ty = kernel.node.ty(self.body, self.tcx); + let kernel_ty = self.monomorphize(kernel_ty); + visit_fn_use(self.tcx, kernel_ty, false, source, &mut self.used_items); + } } mir::TerminatorKind::Drop { ref place, .. } => { let ty = place.ty(self.body, self.tcx).ty; @@ -1475,6 +1484,34 @@ fn collect_roots(tcx: TyCtxt<'_>, mode: MonoItemCollectionStrategy) -> Vec { + for instance in instances { + if instance.def_id().is_local() { + roots.push(dummy_spanned(MonoItem::Fn(instance))); + } + } + } + Err(e) => { + tcx.dcx().emit_err(crate::diagnostics::OffloadManifestReadError { + path: manifest_path.clone(), + err: e.to_string(), + }); + } + } + } + { let entry_fn = tcx.entry_fn(()); @@ -1499,6 +1536,50 @@ fn collect_roots(tcx: TyCtxt<'_>, mode: MonoItemCollectionStrategy) -> Vec( state.visited.into_inner().into_sorted(&mut hcx, true) }); + if tcx + .sess + .opts + .unstable_opts + .offload + .iter() + .any(|o| matches!(o, Offload::Device(p) if p.is_empty())) + { + crate::offload::check_offload_kernels_instantiated(tcx, &mono_items); + } + (mono_items, state.usage_map.into_inner()) } diff --git a/compiler/rustc_monomorphize/src/diagnostics.rs b/compiler/rustc_monomorphize/src/diagnostics.rs index 27705a9837ad3..51ce633f55a54 100644 --- a/compiler/rustc_monomorphize/src/diagnostics.rs +++ b/compiler/rustc_monomorphize/src/diagnostics.rs @@ -50,6 +50,33 @@ pub(crate) struct CouldntDumpMonoStats { pub error: String, } +#[derive(Diagnostic)] +#[diag("could not write offload monomorphization manifest to `{$path}`: {$err}")] +pub(crate) struct OffloadManifestWriteError { + pub path: String, + pub err: String, +} + +#[derive(Diagnostic)] +#[diag("could not read offload monomorphization manifest from `{$path}`: {$err}")] +pub(crate) struct OffloadManifestReadError { + pub path: String, + pub err: String, +} + +#[derive(Diagnostic)] +#[diag("generic offload kernel `{$def_path}` is not instantiated")] +#[help( + "with `-Zoffload=Device` (without a manifest), generic kernels are only discovered via \ + monomorphization; if this kernel is called from host code, pass \ + `-Zoffload=Device=`, using the manifest written by `-Zoffload=HostMetadata=`" +)] +pub(crate) struct GenericKernelNotInstantiated { + #[primary_span] + pub span: Span, + pub def_path: String, +} + #[derive(Diagnostic)] #[diag("the above error was encountered while instantiating `{$kind} {$instance}`")] pub(crate) struct EncounteredErrorWhileInstantiating<'tcx> { diff --git a/compiler/rustc_monomorphize/src/lib.rs b/compiler/rustc_monomorphize/src/lib.rs index c72ee9dd23393..79abdee53eda4 100644 --- a/compiler/rustc_monomorphize/src/lib.rs +++ b/compiler/rustc_monomorphize/src/lib.rs @@ -16,9 +16,14 @@ mod collector; mod diagnostics; mod graph_checks; mod mono_checks; +mod offload; mod partitioning; mod util; +// Exposed so `rustc_codegen_ssa::base::codegen_crate` can trigger the +// host-metadata manifest write. +pub use offload::manifest::write_host_metadata_offload_manifest; + fn custom_coerce_unsize_info<'tcx>( tcx: TyCtxtAt<'tcx>, source_ty: Ty<'tcx>, diff --git a/compiler/rustc_monomorphize/src/offload/manifest.rs b/compiler/rustc_monomorphize/src/offload/manifest.rs new file mode 100644 index 0000000000000..99ed31f0a408d --- /dev/null +++ b/compiler/rustc_monomorphize/src/offload/manifest.rs @@ -0,0 +1,438 @@ +//! Offload manifest: communicates required generic kernel instantiations +//! between host-metadata and device compilation passes. +//! +//! Uses `TyEncoder`/`TyDecoder` to serialize `ty::Instance`. DefIds are +//! encoded as (crate name, DefPath) pairs for stability. + +use std::fs; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lock; +use rustc_hir::def_id::{DefId, DefIndex, LOCAL_CRATE, StableCrateId}; +use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mono::MonoItem; +use rustc_middle::ty::codec::{TyDecoder, TyEncoder}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_serialize::opaque::{FileEncoder, MemDecoder}; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use rustc_span::{ + BlobDecoder, BytePos, ByteSymbol, Pos, Span, SpanDecoder, SpanEncoder, Symbol, SyntaxContext, +}; + +pub(crate) struct OffloadManifestEncoder<'a, 'tcx> { + encoder: FileEncoder<'a>, + type_shorthands: FxHashMap, usize>, + predicate_shorthands: FxHashMap, usize>, + tcx: TyCtxt<'tcx>, +} + +impl<'a, 'tcx> OffloadManifestEncoder<'a, 'tcx> { + pub(crate) fn new(path: &'a std::path::Path, tcx: TyCtxt<'tcx>) -> std::io::Result { + let encoder = FileEncoder::new(path)?; + Ok(OffloadManifestEncoder { + encoder, + type_shorthands: FxHashMap::default(), + predicate_shorthands: FxHashMap::default(), + tcx, + }) + } + + pub(crate) fn finish(mut self) -> std::io::Result<()> { + self.encoder.finish().map(|_| ()).map_err(|(_, e)| e) + } +} + +impl<'a, 'tcx> Encoder for OffloadManifestEncoder<'a, 'tcx> { + fn emit_usize(&mut self, v: usize) { + self.encoder.emit_usize(v); + } + fn emit_u128(&mut self, v: u128) { + self.encoder.emit_u128(v); + } + fn emit_u64(&mut self, v: u64) { + self.encoder.emit_u64(v); + } + fn emit_u32(&mut self, v: u32) { + self.encoder.emit_u32(v); + } + fn emit_u16(&mut self, v: u16) { + self.encoder.emit_u16(v); + } + fn emit_u8(&mut self, v: u8) { + self.encoder.emit_u8(v); + } + fn emit_isize(&mut self, v: isize) { + self.encoder.emit_isize(v); + } + fn emit_i128(&mut self, v: i128) { + self.encoder.emit_i128(v); + } + fn emit_i64(&mut self, v: i64) { + self.encoder.emit_i64(v); + } + fn emit_i32(&mut self, v: i32) { + self.encoder.emit_i32(v); + } + fn emit_i16(&mut self, v: i16) { + self.encoder.emit_i16(v); + } + fn emit_i8(&mut self, v: i8) { + self.encoder.emit_i8(v); + } + fn emit_raw_bytes(&mut self, v: &[u8]) { + self.encoder.emit_raw_bytes(v); + } +} + +impl<'a, 'tcx> SpanEncoder for OffloadManifestEncoder<'a, 'tcx> { + fn encode_span(&mut self, _span: Span) { + // Spans are not needed in the manifest, encode a dummy span. + self.emit_usize(0); + self.emit_usize(0); + self.emit_u32(0); + } + + fn encode_symbol(&mut self, sym: rustc_span::Symbol) { + sym.as_str().encode(self); + } + + fn encode_byte_symbol(&mut self, byte_sym: ByteSymbol) { + let bytes = byte_sym.as_byte_str(); + debug_assert!( + bytes.is_empty(), + "ByteSymbols with content are not expected in offload manifests" + ); + self.emit_usize(bytes.len()); + self.emit_raw_bytes(bytes.as_ref()) + } + + fn encode_expn_id(&mut self, _expn_id: rustc_span::ExpnId) { + self.emit_u32(0); + } + + fn encode_syntax_context(&mut self, _syntax_context: SyntaxContext) { + self.emit_u32(0); + } + + fn encode_crate_num(&mut self, crate_num: rustc_span::def_id::CrateNum) { + self.tcx.stable_crate_id(crate_num).encode(self); + } + + fn encode_def_index(&mut self, def_index: rustc_span::def_id::DefIndex) { + def_index.as_u32().encode(self); + } + + fn encode_def_id(&mut self, def_id: rustc_span::def_id::DefId) { + let crate_name = self.tcx.crate_name(def_id.krate); + let def_path = self.tcx.def_path(def_id); + crate_name.encode(self); + def_path.to_string_no_crate_verbose().encode(self); + } +} + +impl<'a, 'tcx> TyEncoder<'tcx> for OffloadManifestEncoder<'a, 'tcx> { + const CLEAR_CROSS_CRATE: bool = true; + + fn position(&self) -> usize { + self.encoder.position() + } + + fn type_shorthands(&mut self) -> &mut FxHashMap, usize> { + &mut self.type_shorthands + } + + fn predicate_shorthands(&mut self) -> &mut FxHashMap, usize> { + &mut self.predicate_shorthands + } + + fn encode_alloc_id(&mut self, _alloc_id: &rustc_middle::mir::interpret::AllocId) { + // AllocIds are not expected in the manifest. + } +} + +const UNRESOLVED_DEF_ID: DefId = DefId { + krate: rustc_span::def_id::CrateNum::MAX, + index: rustc_span::def_id::DefIndex::from_u32(0), +}; + +/// Decoder used to read the offload monomorphization manifest. +pub(crate) struct OffloadManifestDecoder<'a, 'tcx> { + decoder: MemDecoder<'a>, + type_shorthands: Lock>>, + #[allow(dead_code)] + predicate_shorthands: Lock>>, + tcx: TyCtxt<'tcx>, + /// Map from (crate name, DefPath string) to DefId, used to resolve DefIds + /// across compilation sessions where StableCrateId differs. + def_path_map: Lock>>, +} + +impl<'a, 'tcx> OffloadManifestDecoder<'a, 'tcx> { + pub(crate) fn new(data: &'a [u8], tcx: TyCtxt<'tcx>) -> Result { + let decoder = MemDecoder::new(data, 0)?; + Ok(OffloadManifestDecoder { + decoder, + type_shorthands: Lock::new(FxHashMap::default()), + predicate_shorthands: Lock::new(FxHashMap::default()), + tcx, + def_path_map: Lock::new(None), + }) + } + + /// (crate name, DefPath) -> DefId map for resolving cross-session DefIds. + fn get_or_build_def_path_map(&self) -> FxHashMap<(Symbol, String), DefId> { + let mut guard = self.def_path_map.lock(); + if let Some(map) = guard.as_ref() { + return map.clone(); + } + let map = Self::build_def_path_map(self.tcx); + *guard = Some(map.clone()); + map + } + + /// Build a (crate name, DefPath) -> DefId map. Owns the format details. + fn build_def_path_map(tcx: TyCtxt<'tcx>) -> FxHashMap<(Symbol, String), DefId> { + let mut map: FxHashMap<(Symbol, String), DefId> = FxHashMap::default(); + + let local_crate_name = tcx.crate_name(LOCAL_CRATE); + let krate_items = tcx.hir_crate_items(()); + let local_def_ids = krate_items + .free_items() + .map(|id| id.owner_id.to_def_id()) + .chain(krate_items.trait_items().map(|id| id.owner_id.to_def_id())) + .chain(krate_items.impl_items().map(|id| id.owner_id.to_def_id())) + .chain(krate_items.foreign_items().map(|id| id.owner_id.to_def_id())); + for item_id in local_def_ids { + let def_id = item_id; + let def_path = tcx.def_path(def_id); + map.insert((local_crate_name, def_path.to_string_no_crate_verbose()), def_id); + } + + for &cnum in tcx.crates(()) { + if cnum == LOCAL_CRATE { + continue; + } + let crate_name = tcx.crate_name(cnum); + let num_defs = tcx.num_extern_def_ids(cnum); + for i in 0..num_defs { + let def_id = DefId { krate: cnum, index: DefIndex::from_usize(i) }; + let def_path = tcx.def_path(def_id); + map.entry((crate_name, def_path.to_string_no_crate_verbose())).or_insert(def_id); + } + } + + map + } +} + +impl<'a, 'tcx> Decoder for OffloadManifestDecoder<'a, 'tcx> { + fn read_usize(&mut self) -> usize { + self.decoder.read_usize() + } + fn read_u128(&mut self) -> u128 { + self.decoder.read_u128() + } + fn read_u64(&mut self) -> u64 { + self.decoder.read_u64() + } + fn read_u32(&mut self) -> u32 { + self.decoder.read_u32() + } + fn read_u16(&mut self) -> u16 { + self.decoder.read_u16() + } + fn read_u8(&mut self) -> u8 { + self.decoder.read_u8() + } + fn read_isize(&mut self) -> isize { + self.decoder.read_isize() + } + fn read_i128(&mut self) -> i128 { + self.decoder.read_i128() + } + fn read_i64(&mut self) -> i64 { + self.decoder.read_i64() + } + fn read_i32(&mut self) -> i32 { + self.decoder.read_i32() + } + fn read_i16(&mut self) -> i16 { + self.decoder.read_i16() + } + fn read_i8(&mut self) -> i8 { + self.decoder.read_i8() + } + fn read_raw_bytes(&mut self, len: usize) -> &[u8] { + self.decoder.read_raw_bytes(len) + } + fn peek_byte(&self) -> u8 { + self.decoder.peek_byte() + } + fn position(&self) -> usize { + self.decoder.position() + } +} + +impl<'a, 'tcx> BlobDecoder for OffloadManifestDecoder<'a, 'tcx> { + fn decode_symbol(&mut self) -> rustc_span::Symbol { + let s: String = Decodable::decode(self); + rustc_span::Symbol::intern(&s) + } + + fn decode_byte_symbol(&mut self) -> ByteSymbol { + let len = self.read_usize(); + let bytes = self.read_raw_bytes(len); + ByteSymbol::intern(bytes) + } + + fn decode_def_index(&mut self) -> rustc_span::def_id::DefIndex { + let v = self.read_u32(); + rustc_span::def_id::DefIndex::from_u32(v) + } +} + +impl<'a, 'tcx> SpanDecoder for OffloadManifestDecoder<'a, 'tcx> { + fn decode_span(&mut self) -> Span { + let lo = self.read_usize(); + let hi = self.read_usize(); + let _ctxt = self.read_u32(); + Span::new(BytePos::from_usize(lo), BytePos::from_usize(hi), SyntaxContext::root(), None) + } + + fn decode_expn_id(&mut self) -> rustc_span::ExpnId { + let _ = self.read_u32(); + rustc_span::ExpnId::root() + } + + fn decode_syntax_context(&mut self) -> SyntaxContext { + let _ = self.read_u32(); + SyntaxContext::root() + } + + fn decode_crate_num(&mut self) -> rustc_span::def_id::CrateNum { + let stable_id: StableCrateId = Decodable::decode(self); + self.tcx.stable_crate_id_to_crate_num(stable_id) + } + + fn decode_def_id(&mut self) -> rustc_span::def_id::DefId { + let crate_name: String = Decodable::decode(self); + let crate_name = Symbol::intern(&crate_name); + let def_path_str: String = Decodable::decode(self); + let map = self.get_or_build_def_path_map(); + map.get(&(crate_name, def_path_str)).copied().unwrap_or(UNRESOLVED_DEF_ID) + } + + fn decode_attr_id(&mut self) -> rustc_ast::AttrId { + self.tcx.dcx().fatal("AttrIds are not expected in offload manifests"); + } +} + +impl<'a, 'tcx> TyDecoder<'tcx> for OffloadManifestDecoder<'a, 'tcx> { + const CLEAR_CROSS_CRATE: bool = true; + + fn cached_ty_for_shorthand(&mut self, shorthand: usize, or_insert_with: F) -> Ty<'tcx> + where + F: FnOnce(&mut Self) -> Ty<'tcx>, + { + if let Some(ty) = self.type_shorthands.lock().get(&shorthand) { + return *ty; + } + let ty = or_insert_with(self); + self.type_shorthands.lock().insert(shorthand, ty); + ty + } + + fn with_position(&mut self, pos: usize, f: F) -> R + where + F: FnOnce(&mut Self) -> R, + { + let new_decoder = self.decoder.split_at(pos); + let old_decoder = std::mem::replace(&mut self.decoder, new_decoder); + let result = f(self); + self.decoder = old_decoder; + result + } + + fn decode_alloc_id(&mut self) -> rustc_middle::mir::interpret::AllocId { + self.tcx.dcx().fatal("AllocIds are not expected in offload manifests"); + } +} + +impl<'a, 'tcx> rustc_middle::ty::InternerDecoder for OffloadManifestDecoder<'a, 'tcx> { + type Interner = TyCtxt<'tcx>; + + #[inline] + fn interner(&self) -> Self::Interner { + self.tcx + } +} + +/// Write a list of offload kernel instances to the manifest file. +pub(crate) fn write_manifest<'tcx>( + path: &std::path::Path, + tcx: TyCtxt<'tcx>, + instances: &[ty::Instance<'tcx>], +) -> std::io::Result<()> { + let mut encoder = OffloadManifestEncoder::new(path, tcx)?; + instances.encode(&mut encoder); + encoder.finish() +} + +/// Write out the offload host-metadata manifest for `mono_items`. No-op unless +/// the session was invoked with `-Zoffload=HostMetadata=`. +pub fn write_host_metadata_offload_manifest<'tcx>(tcx: TyCtxt<'tcx>) { + let Some(path) = tcx.sess.opts.unstable_opts.offload.iter().find_map(|o| { + if let rustc_session::config::Offload::HostMetadata(p) = o { Some(p) } else { None } + }) else { + bug!("HostMetadata path not found; caller should have checked"); + }; + + let partitions = tcx.collect_and_partition_mono_items(()); + let mono_items: Vec> = partitions + .codegen_units + .iter() + .flat_map(|cgu| cgu.items().iter()) + .map(|(item, _)| *item) + .collect(); + + let instances: Vec> = mono_items + .iter() + .filter_map(|item| { + if let MonoItem::Fn(instance) = item { + if tcx + .codegen_fn_attrs(instance.def_id()) + .flags + .contains(CodegenFnAttrFlags::OFFLOAD_KERNEL) + { + Some(*instance) + } else { + None + } + } else { + None + } + }) + .collect(); + + if let Err(e) = write_manifest(std::path::Path::new(path), tcx, &instances) { + tcx.dcx().emit_fatal(crate::diagnostics::OffloadManifestWriteError { + path: path.clone(), + err: e.to_string(), + }); + } +} + +/// Read a list of offload kernel instances from the manifest file. +pub(crate) fn read_manifest<'tcx>( + path: &std::path::Path, + tcx: TyCtxt<'tcx>, +) -> std::io::Result>> { + let data = fs::read(path)?; + let mut decoder = OffloadManifestDecoder::new(&data, tcx) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid manifest"))?; + + let instances: Vec> = Decodable::decode(&mut decoder); + + Ok(instances) +} diff --git a/compiler/rustc_monomorphize/src/offload/mod.rs b/compiler/rustc_monomorphize/src/offload/mod.rs new file mode 100644 index 0000000000000..a4dca281e0122 --- /dev/null +++ b/compiler/rustc_monomorphize/src/offload/mod.rs @@ -0,0 +1,46 @@ +pub(crate) mod manifest; + +use rustc_data_structures::fx::FxHashSet; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mono::MonoItem; +use rustc_middle::ty::TyCtxt; + +pub(crate) fn check_offload_kernels_instantiated<'tcx>( + tcx: TyCtxt<'tcx>, + mono_items: &[MonoItem<'tcx>], +) { + let instantiated: FxHashSet = mono_items + .iter() + .filter_map(|item| match item { + MonoItem::Fn(instance) => Some(instance.def_id()), + MonoItem::Static(def_id) => Some(*def_id), + _ => None, + }) + .collect(); + + let crate_items = tcx.hir_crate_items(()); + let check = |def_id: DefId| { + if !matches!(tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) + || !tcx.generics_of(def_id).requires_monomorphization(tcx) + || !tcx.codegen_fn_attrs(def_id).flags.intersects(CodegenFnAttrFlags::OFFLOAD_KERNEL) + || instantiated.contains(&def_id) + { + return; + } + tcx.dcx().emit_err(crate::diagnostics::GenericKernelNotInstantiated { + span: tcx.def_span(def_id), + def_path: tcx.def_path_str(def_id), + }); + }; + for id in crate_items.free_items() { + check(id.owner_id.to_def_id()); + } + for id in crate_items.impl_items() { + check(id.owner_id.to_def_id()); + } + for id in crate_items.trait_items() { + check(id.owner_id.to_def_id()); + } +} diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index cdd18654f0930..1b3e411312ca6 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -844,6 +844,16 @@ fn mono_item_visibility<'tcx>( | InstanceKind::Shim(ShimKind::FnPtrAddr(..)) => return Visibility::Hidden, }; + let attrs = tcx.codegen_fn_attrs(def_id); + if attrs.flags.intersects(CodegenFnAttrFlags::OFFLOAD_KERNEL) { + *can_be_internalized = false; + return default_visibility( + tcx, + def_id, + instance.args.non_erasable_generics().next().is_some(), + ); + } + // Both the `start_fn` lang item and `main` itself should not be exported, // so we give them with `Hidden` visibility but these symbols are // only referenced from the actual `main` symbol which we unfortunately diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 0303081e2c627..efbffa8c0e486 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -196,12 +196,20 @@ pub enum CoverageLevel { // The different settings that the `-Z offload` flag can have. #[derive(Clone, PartialEq, Hash, Debug, Encodable, Decodable)] pub enum Offload { - /// Entry point for `std::offload`, enables kernel compilation for a gpu device - Device, - /// Second step in the offload pipeline, generates the host code to call kernels. + /// Second step in the offload pipeline, enables kernel compilation for a gpu device + /// Reads a manifest of required generic kernel instantiations + /// produced by a previous `HostMetadata` pass. An empty manifest + /// means there are no generic kernels at all, or that generic kernels are only + /// called from non-generic device entry points and never from the host, so we + /// don't need to track their instantiations. + Device(String), + /// Third step in the offload pipeline, generates the host code to call kernels. Host(String), /// Test is similar to Host, but allows testing without a device artifact. Test, + /// First step in the offload pipeline: compile for the host but only emit a manifest of + /// kernel instantiations required by the host code. + HostMetadata(String), } /// The different settings that the `-Z codegen-emit-retag` flag can have. diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 20d1ff55eab6e..3c3f1d69fe2c1 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -819,8 +819,7 @@ mod desc { "a comma-separated list of strings, with elements beginning with + or -"; pub(crate) const parse_pointer_authentication_list_with_polarity: &str = "a comma-separated list of options, each of the form `+` or `-`, where `` is one of: `aarch64-jump-table-hardening`, `auth-traps`, `calls`, `elf-got`, `function-pointer-type-discrimination`, `indirect-gotos`, `init-fini`, `init-fini-address-discrimination`, `intrinsics`, `return-addresses`, `typeinfo-vt-ptr-discrimination`, `vt-ptr-addr-discrimination` or `vt-ptr-type-discrimination`"; pub(crate) const parse_autodiff: &str = "a comma separated list of settings: `Enable`, `PrintSteps`, `PrintTA`, `PrintTAFn`, `PrintAA`, `PrintPerf`, `PrintModBefore`, `PrintModAfter`, `PrintModFinal`, `PrintPasses`, `NoPostopt`, `LooseTypes`, `Inline`, `NoTT`"; - pub(crate) const parse_offload: &str = - "a comma separated list of settings: `Host=`, `Device`, `Test`"; + pub(crate) const parse_offload: &str = "a comma separated list of settings: `Host=`, `HostMetadata=`, `Device` (empty manifest) or `Device=`, `Test`"; pub(crate) const parse_comma_list: &str = "a comma-separated list of strings"; pub(crate) const parse_opt_comma_list: &str = parse_comma_list; pub(crate) const parse_number: &str = "a number"; @@ -1514,12 +1513,17 @@ pub mod parse { return false; } } - "Device" => { - if let Some(_) = arg { - // Device does not accept a value + "HostMetadata" => { + if let Some(p) = arg { + Offload::HostMetadata(p.to_string()) + } else { return false; } - Offload::Device + } + "Device" => { + // Without an argument, `Device` uses an empty manifest and all kernel + // instantiations are discovered via monomorphization. + Offload::Device(arg.unwrap_or_default().to_string()) } "Test" => { if let Some(_) = arg { diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs index 482848578a81b..93db093f630bd 100644 --- a/compiler/rustc_symbol_mangling/src/lib.rs +++ b/compiler/rustc_symbol_mangling/src/lib.rs @@ -106,6 +106,13 @@ pub mod test; pub use v0::{mangle_cgu, mangle_internal_symbol}; +/// Offload kernels need custom v0 symbol treatment because the host +/// and device compilation passes run with different `stable_crate_id`s +/// so they cannot rely on the regular export-hash path. +pub fn is_offload_kernel(attrs: &CodegenFnAttrs) -> bool { + attrs.flags.intersects(CodegenFnAttrFlags::OFFLOAD_KERNEL) +} + /// This function computes the symbol name for the given `instance` and the /// given instantiating crate. That is, if you know that instance X is /// instantiated in crate Y, this is the symbol name this instance would have. @@ -121,6 +128,18 @@ pub fn provide(providers: &mut Providers) { *providers = Providers { symbol_name: symbol_name_provider, ..*providers }; } +/// Compute the v0 symbol name for an offload kernel instance. Forces +/// `is_exportable: true` to omit the `stable_crate_id` disambiguator +/// (which differs between host and device passes). +pub fn mangle_offload_export<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> String { + let instantiating_crate = if is_generic(instance) { + Some(instance.upstream_monomorphization(tcx).unwrap_or(LOCAL_CRATE)) + } else { + None + }; + v0::mangle(tcx, instance, instantiating_crate, true) +} + // The `symbol_name` query provides the symbol name for calling a given // instance from the local crate. In particular, it will also look up the // correct symbol name of instances from upstream crates. @@ -293,45 +312,54 @@ fn compute_symbol_name<'tcx>( tcx.symbol_mangling_version(mangling_version_crate) }; - let symbol = match tcx.is_exportable(def_id) { - true => format!( - "{}.{}", - v0::mangle(tcx, instance, instantiating_crate, true), - export::compute_hash_of_export_fn(tcx, instance) - ), - false => match mangling_version { - SymbolManglingVersion::Legacy => { - let mangled_name = legacy::mangle(tcx, instance, instantiating_crate); - - let mangled_name_too_long = { - // The PDB debug info format cannot store mangled symbol names for which its - // internal record exceeds u16::MAX bytes, a limit multiple Rust projects have been - // hitting due to the verbosity of legacy name mangling. Depending on the linker version - // in use, such symbol names can lead to linker crashes or incomprehensible linker error - // about a limit being hit. - // Mangle those symbols with v0 mangling instead, which gives us more room to breathe - // as v0 mangling is more compact. - // Empirical testing has shown the limit for the symbol name to be 65521 bytes; use - // 65000 bytes to leave some room for prefixes / suffixes as well as unknown scenarios - // with a different limit. - const MAX_SYMBOL_LENGTH: usize = 65000; - - tcx.sess.target.uses_pdb_debuginfo() && mangled_name.len() > MAX_SYMBOL_LENGTH - }; - - if mangled_name_too_long { - v0::mangle(tcx, instance, instantiating_crate, false) - } else { - mangled_name + // Offload kernels must omit the stable_crate_id disambiguator because + // host and device passes have different stable_crate_ids. + let is_offload_kernel = tcx.def_kind(def_id).has_codegen_attrs() + && tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::OFFLOAD_KERNEL); + let symbol = if is_offload_kernel { + v0::mangle(tcx, instance, instantiating_crate, true) + } else { + match tcx.is_exportable(def_id) { + true => format!( + "{}.{}", + v0::mangle(tcx, instance, instantiating_crate, true), + export::compute_hash_of_export_fn(tcx, instance) + ), + false => match mangling_version { + SymbolManglingVersion::Legacy => { + let mangled_name = legacy::mangle(tcx, instance, instantiating_crate); + + let mangled_name_too_long = { + // The PDB debug info format cannot store mangled symbol names for which its + // internal record exceeds u16::MAX bytes, a limit multiple Rust projects have been + // hitting due to the verbosity of legacy name mangling. Depending on the linker version + // in use, such symbol names can lead to linker crashes or incomprehensible linker error + // about a limit being hit. + // Mangle those symbols with v0 mangling instead, which gives us more room to breathe + // as v0 mangling is more compact. + // Empirical testing has shown the limit for the symbol name to be 65521 bytes; use + // 65000 bytes to leave some room for prefixes / suffixes as well as unknown scenarios + // with a different limit. + const MAX_SYMBOL_LENGTH: usize = 65000; + + tcx.sess.target.uses_pdb_debuginfo() + && mangled_name.len() > MAX_SYMBOL_LENGTH + }; + + if mangled_name_too_long { + v0::mangle(tcx, instance, instantiating_crate, false) + } else { + mangled_name + } } - } - SymbolManglingVersion::V0 => v0::mangle(tcx, instance, instantiating_crate, false), - SymbolManglingVersion::Hashed => { - hashed::mangle(tcx, instance, instantiating_crate, || { - v0::mangle(tcx, instance, instantiating_crate, false) - }) - } - }, + SymbolManglingVersion::V0 => v0::mangle(tcx, instance, instantiating_crate, false), + SymbolManglingVersion::Hashed => { + hashed::mangle(tcx, instance, instantiating_crate, || { + v0::mangle(tcx, instance, instantiating_crate, false) + }) + } + }, + } }; debug_assert!( diff --git a/tests/codegen-llvm/gpu_offload/control_flow.rs b/tests/codegen-llvm/gpu_offload/control_flow.rs index 605a6f08843c3..da997de53a428 100644 --- a/tests/codegen-llvm/gpu_offload/control_flow.rs +++ b/tests/codegen-llvm/gpu_offload/control_flow.rs @@ -10,6 +10,8 @@ #![feature(core_intrinsics)] #![no_main] +// CHECK: @.offload_sizes.[[K:[^ ]*foo]] = private unnamed_addr constant + // CHECK: define{{( dso_local)?}} void @main() // CHECK-NOT: define // CHECK: %.offload_baseptrs = alloca [1 x ptr], align 8 @@ -18,9 +20,9 @@ // CHECK: br label %bb3 // CHECK-NOT define // CHECK: bb3 -// CHECK: call void @__tgt_target_data_begin_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull @.offload_sizes.foo, ptr nonnull @.offload_maptypes.foo.begin, ptr null, ptr null) -// CHECK: = call i32 @__tgt_target_kernel(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 256, i32 32, ptr nonnull @.foo.region_id, ptr nonnull %kernel_args) -// CHECK-NEXT: call void @__tgt_target_data_end_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull @.offload_sizes.foo, ptr nonnull @.offload_maptypes.foo.end, ptr null, ptr null) +// CHECK: call void @__tgt_target_data_begin_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull @.offload_sizes.[[K]], ptr nonnull @.offload_maptypes.[[K]].begin, ptr null, ptr null) +// CHECK: = call i32 @__tgt_target_kernel(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 256, i32 32, ptr nonnull @.[[K]].region_id, ptr nonnull %kernel_args) +// CHECK-NEXT: call void @__tgt_target_data_end_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull @.offload_sizes.[[K]], ptr nonnull @.offload_maptypes.[[K]].end, ptr null, ptr null) #[unsafe(no_mangle)] unsafe fn main() { let A = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]; diff --git a/tests/codegen-llvm/gpu_offload/slice_host.rs b/tests/codegen-llvm/gpu_offload/slice_host.rs index 0f27821ef765c..dfc7ec545630c 100644 --- a/tests/codegen-llvm/gpu_offload/slice_host.rs +++ b/tests/codegen-llvm/gpu_offload/slice_host.rs @@ -18,10 +18,10 @@ // CHECK: define{{( dso_local)?}} void @main() // CHECK: %.offload_sizes = alloca [2 x i64], align 8 -// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr {{.*}} %.offload_sizes, ptr {{.*}} @.offload_sizes.foo, i64 16, i1 false) +// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr {{.*}} %.offload_sizes, ptr {{.*}} @.offload_sizes.[[K]], i64 16, i1 false) // CHECK: store i64 16, ptr %.offload_sizes, align 8 // CHECK: call void @__tgt_target_data_begin_mapper(ptr nonnull @anon.[[ID]].1, i64 -1, i32 2, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.[[K]].begin, ptr null, ptr null) -// CHECK: call i32 @__tgt_target_kernel(ptr nonnull @anon.[[ID]].1, i64 -1, i32 1, i32 1, ptr nonnull @.foo.region_id, ptr nonnull %kernel_args) +// CHECK: call i32 @__tgt_target_kernel(ptr nonnull @anon.[[ID]].1, i64 -1, i32 1, i32 1, ptr nonnull @.[[K]].region_id, ptr nonnull %kernel_args) // CHECK-NEXT: call void @__tgt_target_data_end_mapper(ptr nonnull @anon.[[ID]].1, i64 -1, i32 2, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.[[K]].end, ptr null, ptr null) #[unsafe(no_mangle)] diff --git a/tests/pretty/offload/offload_kernel.device.pp b/tests/pretty/offload/offload_kernel.device.pp index 9c8e6edaf2ed3..6f6c4e9693ddf 100644 --- a/tests/pretty/offload/offload_kernel.device.pp +++ b/tests/pretty/offload/offload_kernel.device.pp @@ -18,7 +18,6 @@ use std::offload::offload_kernel; #[rustc_offload_kernel] -#[unsafe(no_mangle)] unsafe extern "gpu-kernel" fn foo(a: &[f32], b: &[f32], c: *mut f32) { *c = a[0] + b[0]; } diff --git a/tests/pretty/offload/offload_kernel.host.pp b/tests/pretty/offload/offload_kernel.host.pp index cf60ee9f8138b..c35cf9474f709 100644 --- a/tests/pretty/offload/offload_kernel.host.pp +++ b/tests/pretty/offload/offload_kernel.host.pp @@ -17,7 +17,7 @@ use std::offload::offload_kernel; -#[unsafe(no_mangle)] +#[rustc_offload_kernel] #[inline(never)] fn foo(_: &[f32], _: &[f32], _: *mut f32) { diff --git a/tests/run-make/offload-generic-manifest/generic.rs b/tests/run-make/offload-generic-manifest/generic.rs new file mode 100644 index 0000000000000..eb356ad05c574 --- /dev/null +++ b/tests/run-make/offload-generic-manifest/generic.rs @@ -0,0 +1,12 @@ +#![feature(core_intrinsics, rustc_attrs)] +#![allow(internal_features)] +#![cfg_attr(device, no_main)] + +#[rustc_offload_kernel] +fn kernel(x: T) {} + +#[cfg(not(device))] +fn main() { + core::intrinsics::offload::<_, _, ()>(kernel::, [1, 1, 1], [1, 1, 1], 0, (0.0f32,)); + core::intrinsics::offload::<_, _, ()>(kernel::, [1, 1, 1], [1, 1, 1], 0, (0i32,)); +} diff --git a/tests/run-make/offload-generic-manifest/rmake.rs b/tests/run-make/offload-generic-manifest/rmake.rs new file mode 100644 index 0000000000000..01b09ff87d2fa --- /dev/null +++ b/tests/run-make/offload-generic-manifest/rmake.rs @@ -0,0 +1,42 @@ +//@ needs-offload + +// Tests the offload manifest pipeline for generic kernels + +use run_make_support::rustc; +use run_make_support::symbols::object_contains_any_symbol_substring; + +fn main() { + rustc() + .input("generic.rs") + .arg("-Zunstable-options") + .arg("-Zoffload=HostMetadata=generic.manifest") + .arg("-Csymbol-mangling-version=v0") + .arg("-Clto=fat") + .emit("metadata") + .run(); + + rustc() + .input("generic.rs") + .cfg("device") + .arg("-Zunstable-options") + .arg("-Zoffload=Device=generic.manifest") + .arg("-Csymbol-mangling-version=v0") + .arg("-Clto=fat") + .emit("obj") + .run(); + + assert!(object_contains_any_symbol_substring("generic.o", &["6kernelfEB2_"])); + assert!(object_contains_any_symbol_substring("generic.o", &["6kernellEB2_"])); + + let p = rustc() + .input("generic.rs") + .cfg("device") + .arg("-Zunstable-options") + .arg("-Zoffload=Device") + .arg("-Csymbol-mangling-version=v0") + .arg("-Clto=fat") + .emit("obj") + .run_fail(); + assert!(p.stderr_utf8().contains("generic offload kernel")); + assert!(p.stderr_utf8().contains("is not instantiated")); +} diff --git a/tests/ui/offload/check_config.rs b/tests/ui/offload/check_config.rs index 69afe65a308b4..ff145f420e482 100644 --- a/tests/ui/offload/check_config.rs +++ b/tests/ui/offload/check_config.rs @@ -1,6 +1,6 @@ //@ revisions: pass fail //@ no-prefer-dynamic -//@ needs-enzyme +//@ needs-offload //@[pass] build-pass //@[fail] build-fail //@[pass] compile-flags: -Zunstable-options -Zoffload=Device -Clto=fat --emit=metadata diff --git a/tests/ui/offload/duplicate_kernel.rs b/tests/ui/offload/duplicate_kernel.rs new file mode 100644 index 0000000000000..abde76137a37c --- /dev/null +++ b/tests/ui/offload/duplicate_kernel.rs @@ -0,0 +1,22 @@ +//@ compile-flags: -Zunstable-options -Zoffload=Device -Clto=fat -Csymbol-mangling-version=v0 --crate-name collision_kernels_a +//@ build-fail +//@ needs-offload + +// An offload kernel whose mangled symbol collides with another item in the +// same crate must be rejected, just like any other symbol collision. + +#![feature(core_intrinsics, rustc_attrs)] +#![allow(internal_features)] + +#[allow(non_snake_case)] +#[no_mangle] +pub fn _RNvC19collision_kernels_a6kernel(_x: f32) {} + +#[rustc_offload_kernel] +fn kernel(_x: f32) {} +//~^ ERROR symbol `_RNvC19collision_kernels_a6kernel` is already defined + +fn main() { + _RNvC19collision_kernels_a6kernel(0.0); + core::intrinsics::offload::<_, _, ()>(kernel, [1, 1, 1], [1, 1, 1], 0, (0.0f32,)); +} diff --git a/tests/ui/offload/duplicate_kernel.stderr b/tests/ui/offload/duplicate_kernel.stderr new file mode 100644 index 0000000000000..bb0f21b1f1cc3 --- /dev/null +++ b/tests/ui/offload/duplicate_kernel.stderr @@ -0,0 +1,8 @@ +error: symbol `_RNvC19collision_kernels_a6kernel` is already defined + --> $DIR/duplicate_kernel.rs:16:1 + | +LL | fn kernel(_x: f32) {} + | ^^^^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/offload/generic_kernel_not_instantiated.rs b/tests/ui/offload/generic_kernel_not_instantiated.rs new file mode 100644 index 0000000000000..36f4f8c667a54 --- /dev/null +++ b/tests/ui/offload/generic_kernel_not_instantiated.rs @@ -0,0 +1,18 @@ +//@ compile-flags: -Zunstable-options -Zoffload=Device -Clto=fat -Csymbol-mangling-version=v0 +//@ build-fail +//@ needs-offload + +// A generic offload kernel that is never called from host code (and hence +// never monomorphized) cannot be discovered without a manifest: with +// `-Zoffload=Device` (no manifest path), the compiler relies on +// monomorphization to find kernels, so it must reject the kernel rather than +// silently emit no device code for it. + +#![feature(rustc_attrs)] +#![allow(internal_features)] + +#[rustc_offload_kernel] +fn kernel(x: T) {} +//~^ ERROR generic offload kernel `kernel` is not instantiated + +fn main() {} diff --git a/tests/ui/offload/generic_kernel_not_instantiated.stderr b/tests/ui/offload/generic_kernel_not_instantiated.stderr new file mode 100644 index 0000000000000..60af2e9b3b92c --- /dev/null +++ b/tests/ui/offload/generic_kernel_not_instantiated.stderr @@ -0,0 +1,10 @@ +error: generic offload kernel `kernel` is not instantiated + --> $DIR/generic_kernel_not_instantiated.rs:15:1 + | +LL | fn kernel(x: T) {} + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: with `-Zoffload=Device` (without a manifest), generic kernels are only discovered via monomorphization; if this kernel is called from host code, pass `-Zoffload=Device=`, using the manifest written by `-Zoffload=HostMetadata=` + +error: aborting due to 1 previous error +