quickcpplib::_xxx::mem_flush_loads_stores::detail Namespace Reference

Typedefs
using	flush_impl_type = memory_flush(*)(const void *addr, size_t bytes, memory_flush kind)
using	potentially_unknown_jump_ptr = void(*)(const byte *data, size_t bytes)

Functions
QUICKCPPLIB_NOINLINE flush_impl_type	make_flush_impl () noexcept
flush_impl_type	flush_impl () noexcept
potentially_unknown_jump_ptr	potentially_unknown_jump (potentially_unknown_jump_ptr set=nullptr)

Typedef Documentation

flush_impl_type

using quickcpplib::_xxx::mem_flush_loads_stores::detail::flush_impl_type = typedef memory_flush (*)(const void *addr, size_t bytes, memory_flush kind)

potentially_unknown_jump_ptr

using quickcpplib::_xxx::mem_flush_loads_stores::detail::potentially_unknown_jump_ptr = typedef void (*)(const byte *data, size_t bytes)

Function Documentation

make_flush_impl()

QUICKCPPLIB_NOINLINE flush_impl_type quickcpplib::_xxx::mem_flush_loads_stores::detail::make_flush_impl ( )

inlinenoexcept

    {
#if defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
#if !defined(_MSC_VER) || (defined(_MSC_VER) && defined(__clang__) && !defined(__c2__))
      static const auto __cpuidex = [](int *cpuInfo, int func1, int func2) {
        __asm__ __volatile__("cpuid\n\t" : "=a"(cpuInfo[0]), "=b"(cpuInfo[1]), "=c"(cpuInfo[2]), "=d"(cpuInfo[3]) : "a"(func1), "c"(func2));
      };  // NOLINT
      // static constexpr auto _mm_clwb = [](const void *addr) { __asm__ __volatile__("clwb (%0)\n\t" : : "r"(addr)); }; // NOLINT
      static const auto _mm_clwb = [](const void *addr) { __asm__ __volatile__(".byte 0x66, 0x0f, 0xae, 0x30\n\t" : : "a"(addr)); };  // NOLINT
      static const auto _mm_clflushopt = [](const void *addr) { __asm__ __volatile__("clflushopt (%0)\n\t" : : "r"(addr)); };         // NOLINT
      static const auto _mm_clflush = [](const void *addr) { __asm__ __volatile__("clflush (%0)\n\t" : : "r"(addr)); };               // NOLINT
      static const auto _mm_sfence = []() { __asm__ __volatile__("sfence\n\t"); };                                                    // NOLINT
#endif
      int nBuff[4];
      __cpuidex(nBuff, 0x7, 0x0);
      if(nBuff[1] & (1 << 24))  // has CLWB instruction
      {
        return [](const void *addr, size_t bytes, memory_flush kind) -> memory_flush {
          if(kind == memory_flush_retain)
          {
            while(bytes > 0)
            {
              _mm_clwb(addr);
              addr = (void *) ((uintptr_t) addr + 64);
              bytes -= 64;
            }
            _mm_sfence();
            return memory_flush_retain;
          }
          while(bytes > 0)
          {
            _mm_clflushopt(addr);
            addr = (void *) ((uintptr_t) addr + 64);
            bytes -= 64;
          }
          _mm_sfence();
          return memory_flush_evict;
        };
      }
      if(nBuff[1] & (1 << 23))  // has CLFLUSHOPT instruction
      {
        return [](const void *addr, size_t bytes, memory_flush /*unused*/) -> memory_flush {
          while(bytes > 0)
          {
            _mm_clflushopt(addr);
            addr = (void *) ((uintptr_t) addr + 64);
            bytes -= 64;
          }
          _mm_sfence();
          return memory_flush_evict;
        };
      }
      else
      {
        // Use CLFLUSH instruction
        return [](const void *addr, size_t bytes, memory_flush /*unused*/) -> memory_flush {
          while(bytes > 0)
          {
            _mm_clflush(addr);
            addr = (void *) ((uintptr_t) addr + 64);
            bytes -= 64;
          }
          return memory_flush_evict;
        };
      }
#elif defined(__aarch64__) || defined(_M_ARM64)
#if !defined(_MSC_VER) || (defined(_MSC_VER) && defined(__clang__) && !defined(__c2__))
      static const auto _dmb_ish = []() { __asm__ __volatile__("dmb ish" : : : "memory"); };
      static const auto _dc_cvac = [](const void *addr) { __asm__ __volatile__("dc cvac, %0" : : "r"(addr) : "memory"); };
      static const auto _dc_civac = [](const void *addr) { __asm__ __volatile__("dc civac, %0" : : "r"(addr) : "memory"); };
#else
      static const auto _dmb_ish = []() { __dmb(_ARM64_BARRIER_ISH); };
      static const auto _dc_cvac = [](const void *addr) {
        (void) addr;
        abort();  // currently MSVC doesn't have an intrinsic for this, could use __emit()?
      };
      static const auto _dc_civac = [](const void *addr) {
        (void) addr;
        abort();  // currently MSVC doesn't have an intrinsic for this, could use __emit()?
      };
#endif
      return [](const void *addr, size_t bytes, memory_flush kind) -> memory_flush {
        if(kind == memory_flush_retain)
        {
          while(bytes > 0)
          {
            _dc_cvac(addr);
            addr = (void *) ((uintptr_t) addr + 64);
            bytes -= 64;
          }
          _dmb_ish();
          return memory_flush_retain;
        }
        while(bytes > 0)
        {
          _dc_civac(addr);
          addr = (void *) ((uintptr_t) addr + 64);
          bytes -= 64;
        }
        _dmb_ish();
        return memory_flush_evict;
      };
#elif defined(__arm__) || defined(_M_ARM)
#if !defined(_MSC_VER) || (defined(_MSC_VER) && defined(__clang__) && !defined(__c2__))
#undef _MoveToCoprocessor
#define _MoveToCoprocessor(value, coproc, opcode1, crn, crm, opcode2)                                                                                          \
  __asm__ __volatile__("MCR p" #coproc ", " #opcode1 ", %0, c" #crn ", c" #crm ", " #opcode2 : : "r"(value) : "memory");  // NOLINT
#endif
      return [](const void *addr, size_t bytes, memory_flush kind) -> memory_flush {
        if(kind == memory_flush_retain)
        {
          while(bytes > 0)
          {
            // __asm__ __volatile__("MCR p15, 0, %0, c7, c10, 1" : : "r"(addr) : "memory");
            _MoveToCoprocessor(addr, 15, 0, 7, 10, 1);
            addr = (void *) ((uintptr_t) addr + 64);
            bytes -= 64;
          }
          // __asm__ __volatile__("mcr p15, 0, %0, c7, c10, 5" : : "r"(0) : "memory");
          _MoveToCoprocessor(0, 15, 0, 7, 10, 5);
          return memory_flush_retain;
        }
        while(bytes > 0)
        {
          // __asm__ __volatile__("MCR p15, 0, %0, c7, c14, 1" : : "r"(addr) : "memory");
          _MoveToCoprocessor(addr, 15, 0, 7, 14, 1);
          addr = (void *) ((uintptr_t) addr + 64);
          bytes -= 64;
        }
        // __asm__ __volatile__("mcr p15, 0, %0, c7, c10, 5" : : "r"(0) : "memory");
        _MoveToCoprocessor(0, 15, 0, 7, 10, 5);
        return memory_flush_evict;
      };
#else
#error Unsupported platform
#endif
    }

flush_impl()

flush_impl_type quickcpplib::_xxx::mem_flush_loads_stores::detail::flush_impl ( )

inlinenoexcept

    {
      static flush_impl_type f;
      if(f != nullptr)
        return f;
      f = make_flush_impl();
      return f;
    }

potentially_unknown_jump()

potentially_unknown_jump_ptr quickcpplib::_xxx::mem_flush_loads_stores::detail::potentially_unknown_jump ( potentially_unknown_jump_ptr  set = nullptr )

externinline

    {
      static potentially_unknown_jump_ptr f = +[](const byte * /*unused*/, size_t /*unused*/) -> void {};
      if(set != nullptr)
      {
        f = set;
      }
      return f;
    }