x

Author: Rafal Rudnicki

llvm/lib/Target/NVPTX/NVPTXIntrinsics.td

@@ -2159,7 +2159,7 @@ multiclass ATOM2S_impl<string OpStr, string IntTypeStr, string TypeStr,
 
 multiclass F_ATOMIC_3_INTRINSIC_PATTERN<RegTyInfo t, string OpStr, string InstructionName> {
   foreach scope = ["cta", "sys"] in {
-    foreach space = ["gen"] in {
+    foreach space = ["gen", "global", "shared"] in {
       defvar intrinsic = !cast<SDPatternOperator>("int_nvvm_atomic_" # OpStr # "_" # space # "_i_" # scope);
       def : Pat<(t.Ty (intrinsic addr:$addr, t.Ty:$b, t.Ty:$c)),
             (!cast<Instruction>(InstructionName # "_rr") ADDR:$addr, t.Ty:$b, t.Ty:$c, Ordering_not_atomic, !cast<PatLeaf>("Scope_" # scope), !cast<PatLeaf>("AddrSpace_" # space))>;
@@ -2174,6 +2174,46 @@ multiclass F_ATOMIC_3_INTRINSIC_PATTERN<RegTyInfo t, string OpStr, string Instru
             (!cast<Instruction>(InstructionName # "_ii") ADDR:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c), Ordering_not_atomic, !cast<PatLeaf>("Scope_" # scope), !cast<PatLeaf>("AddrSpace_" # space))>;
     }
   }
+
+  // Handle intrinsics without explicit scope (defaulting to gpu scope)
+  foreach space = ["gen", "global", "shared"] in {
+    defvar intrinsic = !cast<SDPatternOperator>("int_nvvm_atomic_" # OpStr # "_" # space # "_i");
+    def : Pat<(t.Ty (intrinsic addr:$addr, t.Ty:$b, t.Ty:$c)),
+          (!cast<Instruction>(InstructionName # "_rr") ADDR:$addr, t.Ty:$b, t.Ty:$c, Ordering_not_atomic, Scope_device, !cast<PatLeaf>("AddrSpace_" # space))>;
+
+    def : Pat<(t.Ty (intrinsic addr:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c)),
+          (!cast<Instruction>(InstructionName # "_ir") ADDR:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c, Ordering_not_atomic, Scope_device, !cast<PatLeaf>("AddrSpace_" # space))>;
+
+    def : Pat<(t.Ty (intrinsic addr:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c))),
+          (!cast<Instruction>(InstructionName # "_ri") ADDR:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c), Ordering_not_atomic, Scope_device, !cast<PatLeaf>("AddrSpace_" # space))>;
+
+    def : Pat<(t.Ty (intrinsic addr:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c))),
+          (!cast<Instruction>(InstructionName # "_ii") ADDR:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c), Ordering_not_atomic, Scope_device, !cast<PatLeaf>("AddrSpace_" # space))>;
+  }
+
+  // Handle intrinsics with memory ordering semantics
+  foreach sem = ["acquire", "release", "acq_rel"] in {
+    foreach scope = ["gpu", "cta", "sys"] in {
+      foreach space = ["gen", "global", "shared"] in {
+        defvar intrinsic = !cast<SDPatternOperator>("int_nvvm_atomic_" # OpStr # "_" # space # "_i_" # sem # !if(!eq(scope, "gpu"), "", "_" # scope));
+        defvar ordering = !cast<PatLeaf>("Ordering_" # !subst("acq_rel", "acquire_release", sem));
+        defvar scopeLeaf = !cast<PatLeaf>("Scope_" # !if(!eq(scope, "gpu"), "device", scope));
+        defvar addrSpaceLeaf = !cast<PatLeaf>("AddrSpace_" # space);
+
+        def : Pat<(t.Ty (intrinsic addr:$addr, t.Ty:$b, t.Ty:$c)),
+              (!cast<Instruction>(InstructionName # "_rr") ADDR:$addr, t.Ty:$b, t.Ty:$c, ordering, scopeLeaf, addrSpaceLeaf)>;
+
+        def : Pat<(t.Ty (intrinsic addr:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c)),
+              (!cast<Instruction>(InstructionName # "_ir") ADDR:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c, ordering, scopeLeaf, addrSpaceLeaf)>;
+
+        def : Pat<(t.Ty (intrinsic addr:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c))),
+              (!cast<Instruction>(InstructionName # "_ri") ADDR:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c), ordering, scopeLeaf, addrSpaceLeaf)>;
+
+        def : Pat<(t.Ty (intrinsic addr:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c))),
+              (!cast<Instruction>(InstructionName # "_ii") ADDR:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c), ordering, scopeLeaf, addrSpaceLeaf)>;
+      }
+    }
+  }
 }
 
 // atom.add