Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
run_type: ${{ steps.jobs.outputs.run_type }}
steps:
- name: Checkout the source code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Test citool
# Only test citool on the auto branch, to reduce latency of the calculate matrix job
# on PR/try builds.
Expand Down Expand Up @@ -117,7 +117,7 @@ jobs:
run: git config --global core.autocrlf false

- name: checkout the source code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 2

Expand Down Expand Up @@ -254,7 +254,7 @@ jobs:
df -h

- name: upload artifacts to github
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
# name is set in previous step
name: ${{ env.DOC_ARTIFACT_NAME }}
Expand Down Expand Up @@ -315,7 +315,7 @@ jobs:
environment: ${{ (github.repository == 'rust-lang/rust' && 'bors') || '' }}
steps:
- name: checkout the source code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 2
# Publish the toolstate if an auto build succeeds (just before push to the default branch)
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/dependencies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: checkout the source code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
submodules: recursive
- name: install the bootstrap toolchain
Expand All @@ -66,7 +66,7 @@ jobs:
run: ./src/tools/update-lockfile.sh

- name: upload Cargo.lock artifact for use in PR
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: Cargo-lock
path: |
Expand All @@ -75,7 +75,7 @@ jobs:
src/tools/rustbook/Cargo.lock
retention-days: 1
- name: upload cargo-update log artifact for use in PR
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cargo-updates
path: cargo_update.log
Expand All @@ -91,7 +91,7 @@ jobs:
pull-requests: write
steps:
- name: checkout the source code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: download Cargo.lock from update job
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ghcr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
# Needed to write to the ghcr.io registry
packages: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/post-merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
permissions:
pull-requests: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# Make sure that we have enough commits to find the parent merge commit.
# Since all merges should be through merge commits, fetching two commits
Expand Down
89 changes: 37 additions & 52 deletions compiler/rustc_data_structures/src/jobserver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,65 +4,50 @@ pub use jobserver_crate::Acquired;
use jobserver_crate::{Client, FromEnv, FromEnvErrorKind, HelperThread};
use parking_lot::{Condvar, Mutex};

fn create_client(limit: usize) -> Result<Client, String> {
// Safety: the checked client construction ensures that the jobserver file descriptors
// (if any) are open and valid. We also try to initialize the jobserver as early as possible
// to avoid unrelated file descriptors with matching values becoming open and valid between
// the process start and the jobserver initialization.
let FromEnv { client, var } = unsafe { Client::from_env_ext(true) };

let error = match client {
Ok(client) => return Ok(client),
Err(e) => e,
};

if matches!(
error.kind(),
FromEnvErrorKind::NoEnvVar
| FromEnvErrorKind::NoJobserver
| FromEnvErrorKind::NegativeFd
| FromEnvErrorKind::Unsupported
) {
return Ok(default_client(limit));
}

// Environment specifies jobserver, but it looks incorrect.
// Safety: `error.kind()` should be `NoEnvVar` if `var == None`.
let (name, value) = var.unwrap();
Err(format!(
"failed to connect to jobserver from environment variable `{name}={:?}`: {error}",
value
))
}

// Creates a new jobserver if there's no inherited one.
fn default_client(limit: usize) -> Client {
let client = Client::new(limit).expect("failed to create jobserver");

// Acquire the single token that is always held by the rustc process.
// This is an equivalent of the single token held by a higher level build tool while running
// this instance of rustc. This token is never released - if we are here, then rustc owns the
// jobserver, it is teared down when rustc exits, and there's no one to return the token to.
client.acquire_raw().ok();

client
}

// We stick the jobserver client into a global and initialize it once, because there could be
// multiple compiler instances in this process, and the jobserver is per-process.
static GLOBAL_CLIENT_CHECKED: OnceLock<Client> = OnceLock::new();
static CLIENT: OnceLock<Client> = OnceLock::new();

/// Initializes a jobserver client for the current rustc process.
/// If inheriting jobserver from the environment fails for some reason, an new jobserver owned by
/// the current rustc process will be created. If the inheritance failure reason is non-benign,
/// the passed callback will be used to report the error.
pub fn initialize_checked(limit: usize, report: impl FnOnce(String)) {
GLOBAL_CLIENT_CHECKED.get_or_init(|| match create_client(limit) {
Ok(client) => client,
Err(e) => {
report(e);
default_client(limit)
pub fn initialize(limit: usize, report: impl FnOnce(String)) {
CLIENT.get_or_init(|| {
// Safety: the checked client construction ensures that the jobserver file descriptors
// (if any) are open and valid. We also try to initialize the jobserver as early as possible
// to avoid unrelated file descriptors with matching values becoming open and valid between
// the process start and the jobserver initialization.
let FromEnv { client, var } = unsafe { Client::from_env_ext(true) };

let error = match client {
Ok(client) => return client,
Err(error) => error,
};

if !matches!(
error.kind(),
FromEnvErrorKind::NoEnvVar
| FromEnvErrorKind::NoJobserver
| FromEnvErrorKind::NegativeFd
| FromEnvErrorKind::Unsupported
) {
// Environment specifies jobserver, but it looks incorrect.
// Can unwrap because `var` is `None` only when the error kind is `NoEnvVar`.
let (name, value) = var.unwrap();
let msg = "failed to connect to jobserver from environment variable";
report(format!("{msg} `{name}={value:?}`: {error}"));
}

// Create a new jobserver if there's no inherited one.
let client = Client::new(limit).expect("failed to create jobserver");
// Acquire the single token that is always held by the rustc process.
// This is an equivalent of the single token held by a higher level build tool while
// running this instance of rustc. This token is never released - if we are here, then
// rustc owns the jobserver, it is teared down when rustc exits, and there's no one to
// return the token to.
client.acquire_raw().ok();
client
});
}

Expand All @@ -82,7 +67,7 @@ pub fn initialize_checked(limit: usize, report: impl FnOnce(String)) {
/// To avoid situations like this use the `jobserver::Proxy` wrapper instead,
/// it will ensure that the last token is never released.
pub fn client() -> Client {
GLOBAL_CLIENT_CHECKED.get().expect("uninitialized jobserver client").clone()
CLIENT.get().expect("uninitialized jobserver client").clone()
}

struct ProxyData {
Expand Down
8 changes: 3 additions & 5 deletions compiler/rustc_interface/src/interface.rs
Original file line number Diff line number Diff line change
Expand Up @@ -378,11 +378,9 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
if let Some(limit) =
config.opts.jobs.frontend.max(config.opts.jobs.backend.map(BackendJobs::value))
{
jobserver::initialize_checked(limit.get(), |err| {
early_dcx
.early_struct_warn(err)
.with_note("the build environment is likely misconfigured")
.emit()
jobserver::initialize(limit.get(), |err| {
let note = "the build environment is likely misconfigured";
early_dcx.early_struct_warn(err).with_note(note).emit()
});
}

Expand Down
13 changes: 4 additions & 9 deletions compiler/rustc_passes/src/check_attr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
self.check_rustc_legacy_const_generics(item, *attr_span, fn_indexes)
}
AttributeKind::Doc(attr) => self.check_doc_attrs(attr, hir_id, target),
AttributeKind::EiiImpls(impls) => self.check_eii_impl(impls, target),
AttributeKind::EiiImpls(impls) => self.check_eii_impl(impls),
AttributeKind::RustcMustImplementOneOf { attr_span, fn_names } => {
self.check_rustc_must_implement_one_of(*attr_span, fn_names, hir_id, target)
}
Expand Down Expand Up @@ -472,15 +472,10 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
}
}

fn check_eii_impl(&self, impls: &[EiiImpl], target: Target) {
/// Checks that each externally implementable item (EII) implementation uses `unsafe`
/// exactly when its declaration requires it.
fn check_eii_impl(&self, impls: &[EiiImpl]) {
for EiiImpl { span, inner_span, resolution, impl_unsafe_span, is_default: _ } in impls {
match target {
Target::Fn | Target::Static => {}
_ => {
self.dcx().emit_err(diagnostics::EiiImplTarget { span: *span });
}
}

let impl_unsafe = match resolution {
EiiImplResolution::Macro(eii_macro) => find_attr!(
self.tcx,
Expand Down
7 changes: 0 additions & 7 deletions compiler/rustc_passes/src/diagnostics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1027,13 +1027,6 @@ pub(crate) enum UnexportableItem<'a> {
},
}

#[derive(Diagnostic)]
#[diag("`eii_macro_for` is only valid on functions and statics")]
pub(crate) struct EiiImplTarget {
#[primary_span]
pub span: Span,
}

#[derive(Diagnostic)]
#[diag("`#[{$name}]` is unsafe to implement")]
pub(crate) struct EiiImplRequiresUnsafe {
Expand Down
87 changes: 77 additions & 10 deletions library/core/src/slice/ascii.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,10 @@ impl [u8] {
return false;
}

#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
#[cfg(any(
all(target_arch = "x86_64", target_feature = "sse2"),
all(target_arch = "aarch64", target_feature = "neon")
))]
{
const CHUNK_SIZE: usize = 16;
// The following function has two invariants:
Expand Down Expand Up @@ -108,7 +111,10 @@ impl [u8] {
///
/// The caller must guarantee that the slices are equal in length, and the
/// slice lengths are greater than or equal to `N` bytes.
#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
#[cfg(any(
all(target_arch = "x86_64", target_feature = "sse2"),
all(target_arch = "aarch64", target_feature = "neon")
))]
#[inline]
const fn eq_ignore_ascii_case_chunks<const N: usize>(&self, other: &[u8]) -> bool {
// FIXME(const-hack): The while-loops that follow should be replaced by
Expand Down Expand Up @@ -451,7 +457,8 @@ pub const fn is_ascii_simple(mut bytes: &[u8]) -> bool {
/// (above) returns true, then we know the answer is false.
#[cfg(not(any(
all(target_arch = "x86_64", target_feature = "sse2"),
all(target_arch = "loongarch64", target_feature = "lsx")
all(target_arch = "loongarch64", target_feature = "lsx"),
all(target_arch = "aarch64", target_feature = "neon")
)))]
#[inline]
#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior
Expand Down Expand Up @@ -584,24 +591,81 @@ fn is_ascii_sse2(bytes: &[u8]) -> bool {
rest.iter().all(|b| b.is_ascii())
}

/// ASCII test optimized to use the `pmovmskb` instruction on `x86-64`.
///
/// Uses explicit SSE2 intrinsics to prevent LLVM from auto-vectorizing with
/// broken AVX-512 code that extracts mask bits one-by-one.
#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
/// Chunk size for NEON vectorized ASCII checking (4x 16-byte loads).
#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
const NEON_CHUNK_SIZE: usize = 64;

/// Width of a single NEON vector, used to vectorize the tail left over by the
/// unrolled `NEON_CHUNK_SIZE` loop.
#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
const NEON_VECTOR_SIZE: usize = 16;

#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
#[inline]
fn is_ascii_neon(bytes: &[u8]) -> bool {
use crate::arch::aarch64::{vld1q_u8, vmaxvq_u8, vorrq_u8};

let (chunks, rest) = bytes.as_chunks::<NEON_CHUNK_SIZE>();

for chunk in chunks {
let ptr = chunk.as_ptr();
// SAFETY: chunk is 64 bytes, and `vld1q_u8` has no alignment requirement.
let max = unsafe {
let a1 = vld1q_u8(ptr);
let a2 = vld1q_u8(ptr.add(16));
let b1 = vld1q_u8(ptr.add(32));
let b2 = vld1q_u8(ptr.add(48));
// OR all chunks - if any byte has high bit set, combined will too.
let combined = vorrq_u8(vorrq_u8(a1, a2), vorrq_u8(b1, b2));
// `vmaxvq_u8` is a horizontal reduction with a longer latency than
// `vorrq_u8`, so it runs once per 64 bytes rather than once per load.
vmaxvq_u8(combined)
};
if max >= 128 {
return false;
}
}

// The unrolled loop above leaves up to 63 bytes, so sweep those a vector at
// a time before falling back to a byte-at-a-time check.
let (vectors, rest) = rest.as_chunks::<NEON_VECTOR_SIZE>();

for vector in vectors {
// SAFETY: vector is 16 bytes, and `vld1q_u8` has no alignment requirement.
let max = unsafe { vmaxvq_u8(vld1q_u8(vector.as_ptr())) };
if max >= 128 {
return false;
}
}

// Handle remaining bytes
rest.iter().all(|b| b.is_ascii())
}

/// Uses explicit SIMD intrinsics to prevent LLVM from auto-vectorizing with
/// broken code (e.g., AVX-512 on x86-64 that extracts mask bits one-by-one).
#[cfg(any(
all(target_arch = "x86_64", target_feature = "sse2"),
all(target_arch = "aarch64", target_feature = "neon")
))]
#[inline]
#[rustc_allow_const_fn_unstable(const_eval_select)]
const fn is_ascii(bytes: &[u8]) -> bool {
const USIZE_SIZE: usize = size_of::<usize>();
const NONASCII_MASK: usize = usize::MAX / 255 * 0x80;

#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
const SIMD_MIN_LEN: usize = SSE2_CHUNK_SIZE;
#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
const SIMD_MIN_LEN: usize = NEON_CHUNK_SIZE;

const_eval_select!(
@capture { bytes: &[u8] } -> bool:
if const {
is_ascii_simple(bytes)
} else {
// For small inputs, use usize-at-a-time processing to avoid SSE2 call overhead.
if bytes.len() < SSE2_CHUNK_SIZE {
if bytes.len() < SIMD_MIN_LEN {
let chunks = bytes.chunks_exact(USIZE_SIZE);
let remainder = chunks.remainder();
for chunk in chunks {
Expand All @@ -613,7 +677,10 @@ const fn is_ascii(bytes: &[u8]) -> bool {
return remainder.iter().all(|b| b.is_ascii());
}

is_ascii_sse2(bytes)
#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
{ is_ascii_sse2(bytes) }
#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
{ is_ascii_neon(bytes) }
}
)
}
Expand Down
Loading
Loading