[SYCL][NFCI] Drop __spirv_ops.hpp from core.hpp

sycl/include/sycl/__spirv/spirv_ops.hpp

@@ -233,10 +233,12 @@ template <typename RetT, class HandleT>
 extern __DPCPP_SYCL_EXTERNAL
     RetT __spirv_ConvertHandleToSampledImageINTEL(HandleT);
 
-#define __SYCL_OpGroupAsyncCopyGlobalToLocal __spirv_GroupAsyncCopy
-#define __SYCL_OpGroupAsyncCopyLocalToGlobal __spirv_GroupAsyncCopy
-
 // Atomic SPIR-V builtins
+// TODO: drop these forward-declarations.
+// As of right know, compiler does not forward-declare long long overloads for
+// these and as such we can't drop anything from here. But ideally, we should
+// rely on the compiler to generate those - that would allow to drop
+// spirv_ops.hpp include from more files.
 #define __SPIRV_ATOMIC_LOAD(AS, Type)                                          \
   extern __DPCPP_SYCL_EXTERNAL Type __spirv_AtomicLoad(AS Type *P, int S,      \
                                                        int O) noexcept;
@@ -792,10 +794,6 @@ extern __DPCPP_SYCL_EXTERNAL __ocl_WPipeTy<dataT>
 __spirv_CreatePipeFromPipeStorage_write(
     const ConstantPipeStorage *Storage) noexcept;
 
-extern __DPCPP_SYCL_EXTERNAL void
-__spirv_ocl_prefetch(const __attribute__((opencl_global)) char *Ptr,
-                     size_t NumBytes) noexcept;
-
 extern __DPCPP_SYCL_EXTERNAL float
 __spirv_ConvertBF16ToFINTEL(uint16_t) noexcept;
 extern __DPCPP_SYCL_EXTERNAL uint16_t
@@ -967,43 +965,4 @@ extern __DPCPP_SYCL_EXTERNAL RetT __spirv_TaskSequenceGetINTEL(
 extern __DPCPP_SYCL_EXTERNAL void __spirv_TaskSequenceReleaseINTEL(
     __spv::__spirv_TaskSequenceINTEL *TaskSequence) noexcept;
 
-#else  // if !__SYCL_DEVICE_ONLY__
-
-template <typename dataT>
-__SYCL_CONVERGENT__ extern __ocl_event_t
-__SYCL_OpGroupAsyncCopyGlobalToLocal(int32_t, dataT *Dest, const dataT *Src,
-                                     size_t NumElements, size_t Stride,
-                                     __ocl_event_t) noexcept {
-  for (size_t i = 0; i < NumElements; i++) {
-    Dest[i] = Src[i * Stride];
-  }
-  // A real instance of the class is not needed, return dummy pointer.
-  return nullptr;
-}
-
-template <typename dataT>
-__SYCL_CONVERGENT__ extern __ocl_event_t
-__SYCL_OpGroupAsyncCopyLocalToGlobal(int32_t, dataT *Dest, const dataT *Src,
-                                     size_t NumElements, size_t Stride,
-                                     __ocl_event_t) noexcept {
-  for (size_t i = 0; i < NumElements; i++) {
-    Dest[i * Stride] = Src[i];
-  }
-  // A real instance of the class is not needed, return dummy pointer.
-  return nullptr;
-}
-
-extern __SYCL_EXPORT void __spirv_ocl_prefetch(const char *Ptr,
-                                               size_t NumBytes) noexcept;
-
-__SYCL_CONVERGENT__ extern __DPCPP_SYCL_EXTERNAL __SYCL_EXPORT void
-__spirv_ControlBarrier(__spv::Scope Execution, __spv::Scope Memory,
-                       uint32_t Semantics) noexcept;
-
-__SYCL_CONVERGENT__ extern __DPCPP_SYCL_EXTERNAL __SYCL_EXPORT void
-__spirv_MemoryBarrier(__spv::Scope Memory, uint32_t Semantics) noexcept;
-
-__SYCL_CONVERGENT__ extern __DPCPP_SYCL_EXTERNAL __SYCL_EXPORT void
-__spirv_GroupWaitEvents(__spv::Scope Execution, uint32_t NumEvents,
-                        __ocl_event_t *WaitEvents) noexcept;
 #endif // !__SYCL_DEVICE_ONLY__

sycl/include/sycl/access/access.hpp

@@ -11,7 +11,6 @@
 #include <sycl/detail/defines_elementary.hpp> // for __SYCL2020_DEPRECATED
 
 #ifdef __SYCL_DEVICE_ONLY__
-#include <sycl/__spirv/spirv_ops.hpp>
 #include <type_traits>
 #endif
 
@@ -319,123 +318,6 @@ template <typename T> struct remove_decoration<const T &> {
 template <typename T>
 using remove_decoration_t = typename remove_decoration<T>::type;
 
-namespace detail {
-#ifdef __SYCL_DEVICE_ONLY__
-inline constexpr bool
-address_space_cast_is_possible(access::address_space Src,
-                               access::address_space Dst) {
-  // constant_space is unique and is not interchangeable with any other.
-  auto constant_space = access::address_space::constant_space;
-  if (Src == constant_space || Dst == constant_space)
-    return Src == Dst;
-
-  auto generic_space = access::address_space::generic_space;
-  if (Src == Dst || Src == generic_space || Dst == generic_space)
-    return true;
-
-  // global_host/global_device could be casted to/from global
-  auto global_space = access::address_space::global_space;
-  auto global_device = access::address_space::ext_intel_global_device_space;
-  auto global_host = access::address_space::ext_intel_global_host_space;
-
-  if (Src == global_space || Dst == global_space) {
-    auto Other = Src == global_space ? Dst : Src;
-    if (Other == global_device || Other == global_host)
-      return true;
-  }
-
-  // No more compatible combinations.
-  return false;
-}
-
-template <access::address_space Space, typename ElementType>
-auto static_address_cast(ElementType *Ptr) {
-  constexpr auto SrcAS = deduce_AS<ElementType *>::value;
-  static_assert(address_space_cast_is_possible(SrcAS, Space));
-
-  using dst_type = typename DecoratedType<
-      std::remove_pointer_t<remove_decoration_t<ElementType *>>, Space>::type *;
-
-  // Note: reinterpret_cast isn't enough for some of the casts between different
-  // address spaces, use C-style cast instead.
-  return (dst_type)Ptr;
-}
-
-// Previous implementation (`castAS`, used in `multi_ptr` ctors among other
-// places), used C-style cast instead of a proper dynamic check for some
-// backends/spaces. `SupressNotImplementedAssert = true` parameter is emulating
-// that previous behavior until the proper support is added for compatibility
-// reasons.
-template <access::address_space Space, bool SupressNotImplementedAssert = false,
-          typename ElementType>
-auto dynamic_address_cast(ElementType *Ptr) {
-  constexpr auto generic_space = access::address_space::generic_space;
-  constexpr auto global_space = access::address_space::global_space;
-  constexpr auto local_space = access::address_space::local_space;
-  constexpr auto private_space = access::address_space::private_space;
-  constexpr auto global_device =
-      access::address_space::ext_intel_global_device_space;
-  constexpr auto global_host =
-      access::address_space::ext_intel_global_host_space;
-
-  constexpr auto SrcAS = deduce_AS<ElementType *>::value;
-  using dst_type = typename DecoratedType<
-      std::remove_pointer_t<remove_decoration_t<ElementType *>>, Space>::type *;
-  using RemoveCvT = std::remove_cv_t<ElementType>;
-
-  if constexpr (!address_space_cast_is_possible(SrcAS, Space)) {
-    return (dst_type) nullptr;
-  } else if constexpr (Space == generic_space) {
-    return (dst_type)Ptr;
-  } else if constexpr (Space == global_space &&
-                       (SrcAS == global_device || SrcAS == global_host)) {
-    return (dst_type)Ptr;
-  } else if constexpr (SrcAS == global_space &&
-                       (Space == global_device || Space == global_host)) {
-#if defined(__ENABLE_USM_ADDR_SPACE__)
-    static_assert(SupressNotImplementedAssert || Space != Space,
-                  "Not supported yet!");
-    return detail::static_address_cast<Space>(Ptr);
-#else
-    // If __ENABLE_USM_ADDR_SPACE__ isn't defined then both
-    // global_device/global_host are just aliases for global_space.
-    static_assert(std::is_same_v<dst_type, ElementType *>);
-    return (dst_type)Ptr;
-#endif
-  } else if constexpr (Space == global_space) {
-    return (dst_type)__spirv_GenericCastToPtrExplicit_ToGlobal(
-        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::CrossWorkgroup);
-  } else if constexpr (Space == local_space) {
-    return (dst_type)__spirv_GenericCastToPtrExplicit_ToLocal(
-        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::Workgroup);
-  } else if constexpr (Space == private_space) {
-    return (dst_type)__spirv_GenericCastToPtrExplicit_ToPrivate(
-        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::Function);
-#if !defined(__ENABLE_USM_ADDR_SPACE__)
-  } else if constexpr (SrcAS == generic_space &&
-                       (Space == global_device || Space == global_host)) {
-    return (dst_type)__spirv_GenericCastToPtrExplicit_ToGlobal(
-        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::CrossWorkgroup);
-#endif
-  } else {
-    static_assert(SupressNotImplementedAssert || Space != Space,
-                  "Not supported yet!");
-    return detail::static_address_cast<Space>(Ptr);
-  }
-}
-#else  // __SYCL_DEVICE_ONLY__
-template <access::address_space Space, typename ElementType>
-auto static_address_cast(ElementType *Ptr) {
-  return Ptr;
-}
-template <access::address_space Space, bool SupressNotImplementedAssert = false,
-          typename ElementType>
-auto dynamic_address_cast(ElementType *Ptr) {
-  return Ptr;
-}
-#endif // __SYCL_DEVICE_ONLY__
-} // namespace detail
-
 #undef __OPENCL_GLOBAL_AS__
 #undef __OPENCL_GLOBAL_DEVICE_AS__
 #undef __OPENCL_GLOBAL_HOST_AS__

sycl/include/sycl/detail/address_space_cast.hpp

@@ -0,0 +1,137 @@
+//==------- address_space_cast.hpp --- Implementation of AS casts ----------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <sycl/__spirv/spirv_types.hpp>
+#include <sycl/access/access.hpp>
+
+#include <type_traits>
+
+namespace sycl {
+inline namespace _V1 {
+
+namespace detail {
+#ifdef __SYCL_DEVICE_ONLY__
+inline constexpr bool
+address_space_cast_is_possible(access::address_space Src,
+                               access::address_space Dst) {
+  // constant_space is unique and is not interchangeable with any other.
+  auto constant_space = access::address_space::constant_space;
+  if (Src == constant_space || Dst == constant_space)
+    return Src == Dst;
+
+  auto generic_space = access::address_space::generic_space;
+  if (Src == Dst || Src == generic_space || Dst == generic_space)
+    return true;
+
+  // global_host/global_device could be casted to/from global
+  auto global_space = access::address_space::global_space;
+  auto global_device = access::address_space::ext_intel_global_device_space;
+  auto global_host = access::address_space::ext_intel_global_host_space;
+
+  if (Src == global_space || Dst == global_space) {
+    auto Other = Src == global_space ? Dst : Src;
+    if (Other == global_device || Other == global_host)
+      return true;
+  }
+
+  // No more compatible combinations.
+  return false;
+}
+
+template <access::address_space Space, typename ElementType>
+auto static_address_cast(ElementType *Ptr) {
+  constexpr auto SrcAS = deduce_AS<ElementType *>::value;
+  static_assert(address_space_cast_is_possible(SrcAS, Space));
+
+  using dst_type = typename DecoratedType<
+      std::remove_pointer_t<remove_decoration_t<ElementType *>>, Space>::type *;
+
+  // Note: reinterpret_cast isn't enough for some of the casts between different
+  // address spaces, use C-style cast instead.
+  return (dst_type)Ptr;
+}
+
+// Previous implementation (`castAS`, used in `multi_ptr` ctors among other
+// places), used C-style cast instead of a proper dynamic check for some
+// backends/spaces. `SupressNotImplementedAssert = true` parameter is emulating
+// that previous behavior until the proper support is added for compatibility
+// reasons.
+template <access::address_space Space, bool SupressNotImplementedAssert = false,
+          typename ElementType>
+auto dynamic_address_cast(ElementType *Ptr) {
+  constexpr auto generic_space = access::address_space::generic_space;
+  constexpr auto global_space = access::address_space::global_space;
+  constexpr auto local_space = access::address_space::local_space;
+  constexpr auto private_space = access::address_space::private_space;
+  constexpr auto global_device =
+      access::address_space::ext_intel_global_device_space;
+  constexpr auto global_host =
+      access::address_space::ext_intel_global_host_space;
+
+  constexpr auto SrcAS = deduce_AS<ElementType *>::value;
+  using dst_type = typename DecoratedType<
+      std::remove_pointer_t<remove_decoration_t<ElementType *>>, Space>::type *;
+  using RemoveCvT = std::remove_cv_t<ElementType>;
+
+  if constexpr (!address_space_cast_is_possible(SrcAS, Space)) {
+    return (dst_type) nullptr;
+  } else if constexpr (Space == generic_space) {
+    return (dst_type)Ptr;
+  } else if constexpr (Space == global_space &&
+                       (SrcAS == global_device || SrcAS == global_host)) {
+    return (dst_type)Ptr;
+  } else if constexpr (SrcAS == global_space &&
+                       (Space == global_device || Space == global_host)) {
+#if defined(__ENABLE_USM_ADDR_SPACE__)
+    static_assert(SupressNotImplementedAssert || Space != Space,
+                  "Not supported yet!");
+    return detail::static_address_cast<Space>(Ptr);
+#else
+    // If __ENABLE_USM_ADDR_SPACE__ isn't defined then both
+    // global_device/global_host are just aliases for global_space.
+    static_assert(std::is_same_v<dst_type, ElementType *>);
+    return (dst_type)Ptr;
+#endif
+  } else if constexpr (Space == global_space) {
+    return (dst_type)__spirv_GenericCastToPtrExplicit_ToGlobal(
+        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::CrossWorkgroup);
+  } else if constexpr (Space == local_space) {
+    return (dst_type)__spirv_GenericCastToPtrExplicit_ToLocal(
+        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::Workgroup);
+  } else if constexpr (Space == private_space) {
+    return (dst_type)__spirv_GenericCastToPtrExplicit_ToPrivate(
+        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::Function);
+#if !defined(__ENABLE_USM_ADDR_SPACE__)
+  } else if constexpr (SrcAS == generic_space &&
+                       (Space == global_device || Space == global_host)) {
+    return (dst_type)__spirv_GenericCastToPtrExplicit_ToGlobal(
+        const_cast<RemoveCvT *>(Ptr), __spv::StorageClass::CrossWorkgroup);
+#endif
+  } else {
+    static_assert(SupressNotImplementedAssert || Space != Space,
+                  "Not supported yet!");
+    return detail::static_address_cast<Space>(Ptr);
+  }
+}
+#else  // __SYCL_DEVICE_ONLY__
+template <access::address_space Space, typename ElementType>
+auto static_address_cast(ElementType *Ptr) {
+  return Ptr;
+}
+template <access::address_space Space, bool SupressNotImplementedAssert = false,
+          typename ElementType>
+auto dynamic_address_cast(ElementType *Ptr) {
+  return Ptr;
+}
+#endif // __SYCL_DEVICE_ONLY__
+} // namespace detail
+
+} // namespace _V1
+} // namespace sycl

sycl/include/sycl/detail/spirv.hpp

@@ -10,6 +10,13 @@
 
 #ifdef __SYCL_DEVICE_ONLY__
 
+// Some __spirv_* inrinsics are automatically forward-declared by the compiler,
+// but not all of them. For example:
+//   __spirv_AtomicStore(unsigned long long*, ...)
+// Therefore, we need the following include to get forward-declarations of those
+// versions.
+#include <sycl/__spirv/spirv_ops.hpp>
+
 #include <sycl/ext/oneapi/experimental/non_uniform_groups.hpp> // for IdToMaskPosition
 
 #if defined(__NVPTX__)

sycl/include/sycl/device_event.hpp

@@ -8,7 +8,6 @@
 
 #pragma once
 
-#include <sycl/__spirv/spirv_ops.hpp>
 #include <sycl/__spirv/spirv_types.hpp>
 
 namespace sycl {
@@ -31,7 +30,12 @@ class device_event {
 
   device_event(__ocl_event_t Event) : m_Event(Event) {}
 
-  void wait() { __spirv_GroupWaitEvents(__spv::Scope::Workgroup, 1, &m_Event); }
+  void wait() {
+    (void)m_Event;
+#ifdef __SYCL_DEVICE_ONLY__
+    __spirv_GroupWaitEvents(__spv::Scope::Workgroup, 1, &m_Event);
+#endif
+  }
 };
 
 } // namespace _V1

sycl/include/sycl/ext/intel/esimd/detail/memory_intrin.hpp

@@ -19,6 +19,10 @@
 #include <sycl/ext/intel/esimd/detail/util.hpp>
 #include <sycl/vector.hpp>
 
+#ifdef __SYCL_DEVICE_ONLY__
+#include <sycl/__spirv/spirv_ops.hpp>
+#endif
+
 #include <cstdint>
 
 namespace sycl {

sycl/include/sycl/ext/oneapi/experimental/address_cast.hpp

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include <sycl/detail/spirv.hpp>
+#include <sycl/detail/address_space_cast.hpp>
 #include <sycl/multi_ptr.hpp>
 
 namespace sycl {