diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h index b7b599e1289e..a533bea6f1e6 100644 --- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h +++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h @@ -175,8 +175,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder { return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false, isVolatile, isNontemporal, /*alignment=*/alignmentAttr, - /*mem_order=*/ - cir::MemOrderAttr{}, + /*sync_scope=*/cir::SyncScopeKindAttr{}, + /*mem_order=*/cir::MemOrderAttr{}, /*tbaa=*/cir::TBAAAttr{}); } diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index 15bd6a3628af..5a549eb8629e 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -533,6 +533,15 @@ def CIR_MemOrder : CIR_I32EnumAttr< I32EnumAttrCase<"SequentiallyConsistent", 5, "seq_cst"> ]>; +//===----------------------------------------------------------------------===// +// C/C++ sync scope definitions +//===----------------------------------------------------------------------===// + +def CIR_SyncScopeKind : CIR_I32EnumAttr<"SyncScopeKind", "memory scope kind", [ + I32EnumAttrCase<"SingleThread", 0, "single_thread">, + I32EnumAttrCase<"System", 1, "system"> +]>; + //===----------------------------------------------------------------------===// // AllocaOp //===----------------------------------------------------------------------===// @@ -670,7 +679,8 @@ def CIR_LoadOp : CIR_Op<"load", [ %4 = cir.load volatile %0 : !cir.ptr, i32 // Others - %x = cir.load align(16) atomic(seq_cst) %0 : !cir.ptr, i32 + %x = cir.load align(16) syncscope(single_thread) atomic(seq_cst) + %0 : !cir.ptr, i32 ``` }]; @@ -679,6 +689,7 @@ def CIR_LoadOp : CIR_Op<"load", [ UnitAttr:$is_volatile, UnitAttr:$is_nontemporal, OptionalAttr:$alignment, + OptionalAttr:$sync_scope, OptionalAttr:$mem_order, OptionalAttr:$tbaa ); @@ -689,6 +700,7 @@ def CIR_LoadOp : CIR_Op<"load", [ (`volatile` $is_volatile^)? (`nontemporal` $is_nontemporal^)? (`align` `(` $alignment^ `)`)? + (`syncscope` `(` $sync_scope^ `)`)? (`atomic` `(` $mem_order^ `)`)? $addr `:` qualified(type($addr)) `,` type($result) attr-dict (`tbaa` `(` $tbaa^ `)`)? @@ -698,7 +710,8 @@ def CIR_LoadOp : CIR_Op<"load", [ // TODO(CIR): The final interface here should include an argument for the // SyncScope::ID. // This should be used over the ODS generated setMemOrder. - void setAtomic(cir::MemOrder order); + void setAtomic(cir::MemOrder order, + cir::SyncScopeKind scope); }]; // FIXME: add verifier. @@ -6094,11 +6107,6 @@ def CIR_AtomicXchg : CIR_Op<"atomic.xchg", [ let hasVerifier = 1; } -def CIR_MemScopeKind : CIR_I32EnumAttr<"MemScopeKind", "memory scope kind", [ - I32EnumAttrCase<"SingleThread", 0, "single_thread">, - I32EnumAttrCase<"System", 1, "system"> -]>; - def CIR_AtomicCmpXchg : CIR_Op<"atomic.cmp_xchg", [ AllTypesMatch<["old", "expected", "desired"]> ]> { @@ -6122,7 +6130,7 @@ def CIR_AtomicCmpXchg : CIR_Op<"atomic.cmp_xchg", [ CIR_AnyType:$desired, Arg:$succ_order, Arg:$fail_order, - OptionalAttr:$syncscope, + OptionalAttr:$sync_scope, OptionalAttr:$alignment, UnitAttr:$weak, UnitAttr:$is_volatile); @@ -6135,7 +6143,7 @@ def CIR_AtomicCmpXchg : CIR_Op<"atomic.cmp_xchg", [ `success` `=` $succ_order `,` `failure` `=` $fail_order `)` - (`syncscope` `(` $syncscope^ `)`)? + (`syncscope` `(` $sync_scope^ `)`)? (`align` `(` $alignment^ `)`)? (`weak` $weak^)? (`volatile` $is_volatile^)? @@ -6165,7 +6173,7 @@ def CIR_AtomicTestAndSetOp : CIR_Op<"atomic.test_and_set"> { let arguments = (ins Arg, "", [MemRead, MemWrite]>:$ptr, Arg:$mem_order, - OptionalAttr:$syncscope, + OptionalAttr:$sync_scope, OptionalAttr:$alignment, UnitAttr:$is_volatile); @@ -6196,7 +6204,7 @@ def CIR_AtomicClearOp : CIR_Op<"atomic.clear"> { let arguments = (ins Arg, "", [MemRead, MemWrite]>:$ptr, Arg:$mem_order, - OptionalAttr:$syncscope, + OptionalAttr:$sync_scope, OptionalAttr:$alignment, UnitAttr:$is_volatile); @@ -6230,11 +6238,11 @@ def CIR_AtomicFence : CIR_Op<"atomic.fence"> { let arguments = (ins Arg:$ordering, - OptionalAttr:$syncscope + OptionalAttr:$sync_scope ); let assemblyFormat = [{ - (`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict + (`syncscope` `(` $sync_scope^ `)`)? $ordering attr-dict }]; } diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp index 54fd8b5219d8..a21805fa1086 100644 --- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp @@ -17,6 +17,7 @@ #include "CIRGenOpenMPRuntime.h" #include "TargetInfo.h" #include "clang/AST/ASTContext.h" +#include "clang/Basic/SyncScope.h" #include "clang/CIR/Dialect/IR/CIRAttrs.h" #include "clang/CIR/Dialect/IR/CIRDataLayout.h" #include "clang/CIR/Dialect/IR/CIRDialect.h" @@ -350,6 +351,20 @@ static cir::IntAttr extractIntAttr(mlir::Value v) { return {}; } +// Maps SyncScope::SingleScope to SyncScopeKind::SingleThread, +// SyncScope::SystemScope to SyncScopeKind::System, +// and asserts (llvm_unreachable) for anything else. +static cir::SyncScopeKind convertSyncScopeToCIR(clang::SyncScope scope) { + switch (scope) { + case clang::SyncScope::SingleScope: + return cir::SyncScopeKind::SingleThread; + case clang::SyncScope::SystemScope: + return cir::SyncScopeKind::System; + default: + llvm_unreachable("NYI"); + } +} + // Inspect a value that is the strong/weak flag for a compare-exchange. If it // is a constant of intergral or boolean type, set `val` to the constant's // boolean value and return true. Otherwise leave `val` unchanged and return @@ -418,7 +433,7 @@ static void emitAtomicCmpXchg(CIRGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Val2, uint64_t Size, cir::MemOrder SuccessOrder, cir::MemOrder FailureOrder, - cir::MemScopeKind Scope) { + cir::SyncScopeKind Scope) { auto &builder = CGF.getBuilder(); auto loc = CGF.getLoc(E->getSourceRange()); auto Expected = builder.createLoad(loc, Val1); @@ -428,7 +443,7 @@ static void emitAtomicCmpXchg(CIRGenFunction &CGF, AtomicExpr *E, bool IsWeak, builder, loc, Expected.getType(), boolTy, Ptr.getPointer(), Expected, Desired, cir::MemOrderAttr::get(&CGF.getMLIRContext(), SuccessOrder), cir::MemOrderAttr::get(&CGF.getMLIRContext(), FailureOrder), - cir::MemScopeKindAttr::get(&CGF.getMLIRContext(), Scope), + cir::SyncScopeKindAttr::get(&CGF.getMLIRContext(), Scope), builder.getI64IntegerAttr(Ptr.getAlignment().getAsAlign().value())); cmpxchg.setIsVolatile(E->isVolatile()); cmpxchg.setWeak(IsWeak); @@ -456,7 +471,7 @@ static void emitAtomicCmpXchg(CIRGenFunction &CGF, AtomicExpr *E, bool IsWeak, static void emitAtomicCmpXchgFailureSet( CIRGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, mlir::Value FailureOrderVal, uint64_t Size, - cir::MemOrder SuccessOrder, cir::MemScopeKind Scope) { + cir::MemOrder SuccessOrder, cir::SyncScopeKind Scope) { cir::MemOrder FailureOrder; if (auto ordAttr = extractIntAttr(FailureOrderVal)) { @@ -546,7 +561,7 @@ static void emitAtomicOp(CIRGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, mlir::Value IsWeak, mlir::Value FailureOrder, uint64_t Size, cir::MemOrder Order, - cir::MemScopeKind Scope) { + cir::SyncScopeKind Scope) { assert(!cir::MissingFeatures::syncScopeID()); StringRef Op; @@ -594,9 +609,7 @@ static void emitAtomicOp(CIRGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__scoped_atomic_load_n: case AtomicExpr::AO__scoped_atomic_load: { auto load = builder.createLoad(loc, Ptr); - // FIXME(cir): add scope information. - assert(!cir::MissingFeatures::syncScopeID()); - load.setMemOrder(Order); + load.setAtomic(Order, Scope); load.setIsVolatile(E->isVolatile()); // TODO(cir): this logic should be part of createStore, but doing so @@ -748,7 +761,7 @@ static void emitAtomicOp(CIRGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__atomic_test_and_set: { auto op = cir::AtomicTestAndSetOp::create( builder, loc, Ptr.getPointer(), Order, - cir::MemScopeKindAttr::get(&CGF.getMLIRContext(), Scope), + cir::SyncScopeKindAttr::get(&CGF.getMLIRContext(), Scope), builder.getI64IntegerAttr(Ptr.getAlignment().getQuantity()), E->isVolatile()); builder.createStore(loc, op, Dest); @@ -758,7 +771,7 @@ static void emitAtomicOp(CIRGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__atomic_clear: { cir::AtomicClearOp::create( builder, loc, Ptr.getPointer(), Order, - cir::MemScopeKindAttr::get(&CGF.getMLIRContext(), Scope), + cir::SyncScopeKindAttr::get(&CGF.getMLIRContext(), Scope), builder.getI64IntegerAttr(Ptr.getAlignment().getQuantity()), E->isVolatile()); return; @@ -813,14 +826,17 @@ static void emitAtomicOp(CIRGenFunction &CGF, AtomicExpr *Expr, Address Dest, if (!ScopeModel) { assert(!cir::MissingFeatures::syncScopeID()); emitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, - Order, cir::MemScopeKind::System); + Order, cir::SyncScopeKind::System); return; } // Handle constant scope. - if (extractIntAttr(Scope)) { + if (auto scopeAttr = extractIntAttr(Scope)) { assert(!cir::MissingFeatures::syncScopeID()); - llvm_unreachable("NYI"); + auto mappedScope = + convertSyncScopeToCIR(ScopeModel->map(scopeAttr.getUInt())); + emitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, + Order, mappedScope); return; } diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h index c9e5996b9360..5b8391c9d408 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h +++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h @@ -870,6 +870,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy { return cir::LoadOp::create( *this, loc, addr.getElementType(), addr.getPointer(), /*isDeref=*/false, /*is_volatile=*/isVolatile, /*is_nontemporal=*/isNontemporal, align, + /*sync_scope=*/cir::SyncScopeKindAttr{}, /*mem_order=*/cir::MemOrderAttr{}, /*tbaa=*/cir::TBAAAttr{}); } diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index ae76246a9204..58ea0e722e47 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -370,7 +370,7 @@ static mlir::Value makeAtomicCmpXchgValue(CIRGenFunction &cgf, cir::MemOrder::SequentiallyConsistent), MemOrderAttr::get(&cgf.getMLIRContext(), cir::MemOrder::SequentiallyConsistent), - MemScopeKindAttr::get(&cgf.getMLIRContext(), cir::MemScopeKind::System), + SyncScopeKindAttr::get(&cgf.getMLIRContext(), cir::SyncScopeKind::System), builder.getI64IntegerAttr(destAddr.getAlignment().getAsAlign().value())); return returnBool ? op.getResult(1) : op.getResult(0); @@ -378,7 +378,7 @@ static mlir::Value makeAtomicCmpXchgValue(CIRGenFunction &cgf, static mlir::Value makeAtomicFenceValue(CIRGenFunction &cgf, const CallExpr *expr, - cir::MemScopeKind syncScope) { + cir::SyncScopeKind syncScope) { auto &builder = cgf.getBuilder(); mlir::Value orderingVal = cgf.emitScalarExpr(expr->getArg(0)); @@ -392,7 +392,7 @@ static mlir::Value makeAtomicFenceValue(CIRGenFunction &cgf, cir::AtomicFence::create( builder, cgf.getLoc(expr->getSourceRange()), ordering, - MemScopeKindAttr::get(&cgf.getMLIRContext(), syncScope)); + SyncScopeKindAttr::get(&cgf.getMLIRContext(), syncScope)); } return mlir::Value(); @@ -2155,10 +2155,10 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__atomic_thread_fence: return RValue::get( - makeAtomicFenceValue(*this, E, cir::MemScopeKind::System)); + makeAtomicFenceValue(*this, E, cir::SyncScopeKind::System)); case Builtin::BI__atomic_signal_fence: return RValue::get( - makeAtomicFenceValue(*this, E, cir::MemScopeKind::SingleThread)); + makeAtomicFenceValue(*this, E, cir::SyncScopeKind::SingleThread)); case Builtin::BI__c11_atomic_thread_fence: case Builtin::BI__c11_atomic_signal_fence: llvm_unreachable("BI__c11_atomic_thread_fence like NYI"); diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp index 65f8dd822399..c8eead655918 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp @@ -18,6 +18,7 @@ #include "CIRGenFunction.h" #include "CIRGenModule.h" #include "TargetInfo.h" +#include "clang/CIR/Dialect/IR/CIROpsEnums.h" #include "clang/CIR/MissingFeatures.h" // TODO(cir): once all builtins are covered, decide whether we still @@ -4489,7 +4490,7 @@ CIRGenFunction::emitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, case NEON::BI__builtin_neon_vldap1q_lane_s64: { cir::LoadOp Load = builder.createAlignedLoad( Ops[0].getLoc(), vTy.getElementType(), Ops[0], PtrOp0.getAlignment()); - Load.setAtomic(cir::MemOrder::Acquire); + Load.setAtomic(cir::MemOrder::Acquire, cir::SyncScopeKind::System); return cir::VecInsertOp::create(builder, getLoc(E->getExprLoc()), builder.createBitcast(Ops[1], vTy), Load, Ops[2]); diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp index 0417f6dfdc1e..1f2d912f6c31 100644 --- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp +++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp @@ -1013,12 +1013,9 @@ LogicalResult cir::ComplexImagPtrOp::verify() { // LoadOp //===----------------------------------------------------------------------===// -// TODO(CIR): The final interface here should include an argument for the -// SyncScope::ID. -void cir::LoadOp::setAtomic(cir::MemOrder order) { +void cir::LoadOp::setAtomic(cir::MemOrder order, cir::SyncScopeKind scope) { setMemOrder(order); - if (cir::MissingFeatures::syncScopeID()) - llvm_unreachable("NYI"); + setSyncScope(scope); } //===----------------------------------------------------------------------===// diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/ItaniumCXXABI.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/ItaniumCXXABI.cpp index 47cb5c1fd7ad..8b333d38cc92 100644 --- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/ItaniumCXXABI.cpp +++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/ItaniumCXXABI.cpp @@ -386,7 +386,9 @@ void ItaniumCXXABI::lowerGetMethod( rewriter, op.getLoc(), vtablePtrPtr, /*isDeref=*/false, /*isVolatile=*/false, /*isNontemporal=*/false, - /*alignment=*/mlir::IntegerAttr(), /*mem_order=*/cir::MemOrderAttr(), + /*alignment=*/mlir::IntegerAttr(), + /*sync_scope=*/cir::SyncScopeKindAttr{}, + /*mem_order=*/cir::MemOrderAttr(), /*tbaa=*/mlir::ArrayAttr()); // Get the vtable offset. @@ -421,6 +423,7 @@ void ItaniumCXXABI::lowerGetMethod( /*isDeref=*/false, /*isVolatile=*/false, /*isNontemporal=*/false, /*alignment=*/mlir::IntegerAttr(), + /*sync_scope=*/cir::SyncScopeKindAttr{}, /*mem_order=*/cir::MemOrderAttr(), /*tbaa=*/mlir::ArrayAttr()); } diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index 9308a278e107..376b4a5c3fc1 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -423,9 +423,9 @@ static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter, } std::optional -getLLVMSyncScope(std::optional syncScope) { +getLLVMSyncScope(std::optional syncScope) { if (syncScope.has_value()) - return syncScope.value() == cir::MemScopeKind::SingleThread ? "singlethread" + return syncScope.value() == cir::SyncScopeKind::SingleThread ? "singlethread" : ""; return std::nullopt; } @@ -1829,11 +1829,13 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite( invariant = isLoadOrStoreInvariant(op.getAddr()); // TODO: nontemporal, syncscope. + auto syncScope = getLLVMSyncScope(op.getSyncScope()); auto newLoad = mlir::LLVM::LoadOp::create( rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), - /* alignment */ alignment, op.getIsVolatile(), - /* nontemporal */ op.getIsNontemporal(), - /* invariant */ false, /* invariantGroup */ invariant, ordering); + /*alignment*/ alignment, op.getIsVolatile(), + /*nontemporal*/ op.getIsNontemporal(), + /*invariant*/ false, /*invariantGroup*/ invariant, ordering, + /*syncscope=*/ syncScope.value_or(StringRef())); // Convert adapted result to its original type if needed. mlir::Value result = @@ -3499,7 +3501,7 @@ mlir::LogicalResult CIRToLLVMAtomicCmpXchgLowering::matchAndRewrite( rewriter, op.getLoc(), adaptor.getPtr(), expected, desired, getLLVMAtomicOrder(adaptor.getSuccOrder()), getLLVMAtomicOrder(adaptor.getFailOrder())); - cmpxchg.setSyncscope(getLLVMSyncScope(adaptor.getSyncscope())); + cmpxchg.setSyncscope(getLLVMSyncScope(adaptor.getSyncScope())); cmpxchg.setAlignment(adaptor.getAlignment()); cmpxchg.setWeak(adaptor.getWeak()); cmpxchg.setVolatile_(adaptor.getIsVolatile()); @@ -3666,7 +3668,7 @@ mlir::LogicalResult CIRToLLVMAtomicTestAndSetOpLowering::matchAndRewrite( mlir::ConversionPatternRewriter &rewriter) const { mlir::LLVM::AtomicOrdering llvmOrder = getLLVMAtomicOrder(op.getMemOrder()); llvm::StringRef llvmSyncScope = - getLLVMSyncScope(adaptor.getSyncscope()).value_or(StringRef()); + getLLVMSyncScope(adaptor.getSyncScope()).value_or(StringRef()); auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), rewriter.getI8Type(), 1); @@ -3702,7 +3704,7 @@ mlir::LogicalResult CIRToLLVMAtomicFenceLowering::matchAndRewrite( auto llvmOrder = getLLVMAtomicOrder(adaptor.getOrdering()); auto fence = mlir::LLVM::FenceOp::create(rewriter, op.getLoc(), llvmOrder); - fence.setSyncscope(getLLVMSyncScope(adaptor.getSyncscope())); + fence.setSyncscope(getLLVMSyncScope(adaptor.getSyncScope())); rewriter.replaceOp(op, fence); diff --git a/clang/lib/CIR/Lowering/ThroughMLIR/LowerCIRLoopToSCF.cpp b/clang/lib/CIR/Lowering/ThroughMLIR/LowerCIRLoopToSCF.cpp index 04749f5d2da2..e38cb4a8171c 100644 --- a/clang/lib/CIR/Lowering/ThroughMLIR/LowerCIRLoopToSCF.cpp +++ b/clang/lib/CIR/Lowering/ThroughMLIR/LowerCIRLoopToSCF.cpp @@ -482,6 +482,7 @@ class CIRWhileOpLowering : public mlir::OpConversionPattern { auto cond = LoadOp::create( rewriter, loc, boolTy, condAlloca, /*isDeref=*/false, /*volatile=*/false, /*nontemporal=*/false, alignment, + /*sync_scope=*/cir::SyncScopeKindAttr{}, /*memorder=*/cir::MemOrderAttr{}, /*tbaa=*/cir::TBAAAttr{}); auto ifnot = IfOp::create(rewriter, loc, cond, /*withElseRegion=*/false, [&](mlir::OpBuilder &, mlir::Location) { diff --git a/clang/test/CIR/CodeGen/AArch64/neon-ldst.c b/clang/test/CIR/CodeGen/AArch64/neon-ldst.c index fb694777299d..aac34e2be6ec 100644 --- a/clang/test/CIR/CodeGen/AArch64/neon-ldst.c +++ b/clang/test/CIR/CodeGen/AArch64/neon-ldst.c @@ -637,7 +637,7 @@ uint64x2_t test_vldap1q_lane_u64(uint64_t *a, uint64x2_t b) { // CIR-LABEL:test_vldap1q_lane_u64 // CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !u64 +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !u64 // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector @@ -654,7 +654,7 @@ int64x2_t test_vldap1q_lane_s64(int64_t *a, int64x2_t b) { // CIR-LABEL:test_vldap1q_lane_s64 // CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector @@ -671,7 +671,7 @@ float64x2_t test_vldap1q_lane_f64(float64_t *a, float64x2_t b) { // CIR-LABEL:test_vldap1q_lane_f64 // CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !cir.double +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !cir.double // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector @@ -688,7 +688,7 @@ poly64x2_t test_vldap1q_lane_p64(poly64_t *a, poly64x2_t b) { // CIR-LABEL:test_vldap1q_lane_p64 // CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector @@ -705,7 +705,7 @@ uint64x1_t test_vldap1_lane_u64(uint64_t *a, uint64x1_t b) { // CIR-LABEL:test_vldap1_lane_u64 // CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !u64 +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !u64 // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector @@ -722,7 +722,7 @@ int64x1_t test_vldap1_lane_s64(int64_t *a, int64x1_t b) { // CIR-LABEL:test_vldap1_lane_s64 // CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector @@ -740,7 +740,7 @@ float64x1_t test_vldap1_lane_f64(float64_t *a, float64x1_t b) { // CIR-LABEL: test_vldap1_lane_f64 // CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !cir.double +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !cir.double // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector @@ -757,7 +757,7 @@ poly64x1_t test_vldap1_lane_p64(poly64_t *a, poly64x1_t b) { // CIR-LABEL: test_vldap1_lane_p64 // CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i // CIR: [[TMP0:%.*]] = cir.cast bitcast {{.*}} : !cir.ptr -> !cir.ptr -// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 +// CIR: [[VAL:%.*]] = cir.load align(8) syncscope(system) atomic(acquire) [[TMP0]] : !cir.ptr, !s64 // CIR: [[VEC:%.*]] = cir.cast bitcast {{.*}} : !cir.vector -> !cir.vector // CIR: [[TMP:%.*]] = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector diff --git a/clang/test/CIR/CodeGen/scoped-atomic-load-store.c b/clang/test/CIR/CodeGen/scoped-atomic-load-store.c new file mode 100644 index 000000000000..a8aa5665ec30 --- /dev/null +++ b/clang/test/CIR/CodeGen/scoped-atomic-load-store.c @@ -0,0 +1,35 @@ +// RUN: %clang_cc1 -x c -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR +// RUN: %clang_cc1 -x c -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll +// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM +// RUN: %clang_cc1 -x c -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll +// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG + +int scoped_load_thread(int *ptr) { + return __scoped_atomic_load_n(ptr, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CIR-LABEL: @scoped_load_thread +// CIR: %[[ATOMIC_LOAD:.*]] = cir.load align(4) syncscope(single_thread) atomic(relaxed) %{{.*}} : !cir.ptr, !s32i +// CIR: cir.store align(4) %[[ATOMIC_LOAD]], %{{.*}} : !s32i, !cir.ptr + +// LLVM-LABEL: @scoped_load_thread +// LLVM: load atomic i32, ptr %{{.*}} syncscope("singlethread") monotonic, align 4 + +// OGCG-LABEL: @scoped_load_thread +// OGCG: load atomic i32, ptr %{{.*}} monotonic, align 4 + +int scoped_load_system(int *ptr) { + return __scoped_atomic_load_n(ptr, __ATOMIC_SEQ_CST, __MEMORY_SCOPE_SYSTEM); +} + +// CIR-LABEL: @scoped_load_system +// CIR: cir.load align(4) syncscope(system) atomic(seq_cst) %{{.*}} : !cir.ptr, !s32i + +// LLVM-LABEL: @scoped_load_system +// LLVM: load atomic i32, ptr %{{.*}} seq_cst, align 4 +// LLVM-NOT: syncscope( + +// OGCG-LABEL: @scoped_load_system +// OGCG: load atomic i32, ptr %{{.*}} seq_cst, align 4 +// OGCG-NOT: syncscope( diff --git a/clang/test/CIR/Lowering/syncscope.cir b/clang/test/CIR/Lowering/syncscope.cir index 8e1aad8a743c..49ef200a1a3a 100644 --- a/clang/test/CIR/Lowering/syncscope.cir +++ b/clang/test/CIR/Lowering/syncscope.cir @@ -1,12 +1,29 @@ -// RUN: cir-translate %s -cir-to-llvmir --disable-cc-lowering -o - | FileCheck %s -check-prefix=LLVM - -!s32i = !cir.int -#fn_attr = #cir, nothrow = #cir.nothrow, optnone = #cir.optnone})> -module { - cir.func @test(%ptr: !cir.ptr, %expected: !s32i, %desired: !s32i) -> !cir.bool extra(#fn_attr) { - %old, %cmp = cir.atomic.cmp_xchg(%ptr : !cir.ptr, %expected : !s32i, %desired : !s32i, success = acquire, failure = acquire) syncscope(single_thread) align(4) : (!s32i, !cir.bool) - cir.return %cmp: !cir.bool - } -} - -// LLVM: {{%.*}} = cmpxchg ptr {{%.*}}, i32 {{%.*}}, i32 {{%.*}} syncscope("singlethread") acquire acquire, align 4 +// RUN: cir-translate %s -cir-to-llvmir --disable-cc-lowering -o - | FileCheck %s -check-prefix=LLVM + +!s32i = !cir.int +#fn_attr = #cir, nothrow = #cir.nothrow, optnone = #cir.optnone})> +module { + cir.func @test(%ptr: !cir.ptr, %expected: !s32i, %desired: !s32i) -> !cir.bool extra(#fn_attr) { + %old, %cmp = cir.atomic.cmp_xchg(%ptr : !cir.ptr, %expected : !s32i, %desired : !s32i, success = acquire, failure = acquire) syncscope(single_thread) align(4) : (!s32i, !cir.bool) + cir.return %cmp: !cir.bool + } + + cir.func @load(%ptr: !cir.ptr) -> !s32i extra(#fn_attr) { + %val = cir.load syncscope(single_thread) atomic(relaxed) %ptr : !cir.ptr, !s32i + cir.return %val : !s32i + } + + + // System scope should not materialize a syncscope attribute. + cir.func @system_load(%ptr: !cir.ptr) -> !s32i extra(#fn_attr) { + %val = cir.load atomic(seq_cst) %ptr : !cir.ptr, !s32i + cir.return %val : !s32i + } + +} + +// LLVM: {{%.*}} = cmpxchg ptr {{%.*}}, i32 {{%.*}}, i32 {{%.*}} syncscope("singlethread") acquire acquire, align 4 +// LLVM: load atomic i32, ptr {{%.*}} syncscope("singlethread") monotonic, align 4 +// LLVM-LABEL: @system_load +// LLVM: load atomic i32, ptr {{%.*}} seq_cst, align 4 +// LLVM-NOT: syncscope(