erlang各种有用的函数包括一些有用nif封装,还有一些性能测试case。
選択できるのは25トピックまでです。 トピックは、先頭が英数字で、英数字とダッシュ('-')を使用した35文字以内のものにしてください。

3957 行
194 KiB

3年前
  1. // Provides a C++11 implementation of a multi-producer, multi-consumer lock-free queue.
  2. // An overview, including benchmark results, is provided here:
  3. // http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++
  4. // The full design is also described in excruciating detail at:
  5. // http://moodycamel.com/blog/2014/detailed-design-of-a-lock-free-queue
  6. // Simplified BSD license:
  7. // Copyright (c) 2013-2020, Cameron Desrochers.
  8. // All rights reserved.
  9. //
  10. // Redistribution and use in source and binary forms, with or without modification,
  11. // are permitted provided that the following conditions are met:
  12. //
  13. // - Redistributions of source code must retain the above copyright notice, this list of
  14. // conditions and the following disclaimer.
  15. // - Redistributions in binary form must reproduce the above copyright notice, this list of
  16. // conditions and the following disclaimer in the documentation and/or other materials
  17. // provided with the distribution.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
  20. // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  21. // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  22. // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
  24. // OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25. // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  26. // TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27. // EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. // Also dual-licensed under the Boost Software License (see LICENSE.md)
  29. #pragma once
  30. #if defined(__GNUC__) && !defined(__INTEL_COMPILER)
  31. // Disable -Wconversion warnings (spuriously triggered when Traits::size_t and
  32. // Traits::index_t are set to < 32 bits, causing integer promotion, causing warnings
  33. // upon assigning any computed values)
  34. #pragma GCC diagnostic push
  35. #pragma GCC diagnostic ignored "-Wconversion"
  36. #ifdef MCDBGQ_USE_RELACY
  37. #pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
  38. #endif
  39. #endif
  40. #if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17)
  41. // VS2019 with /W4 warns about constant conditional expressions but unless /std=c++17 or higher
  42. // does not support `if constexpr`, so we have no choice but to simply disable the warning
  43. #pragma warning(push)
  44. #pragma warning(disable: 4127) // conditional expression is constant
  45. #endif
  46. #if defined(__APPLE__)
  47. #include "TargetConditionals.h"
  48. #endif
  49. #ifdef MCDBGQ_USE_RELACY
  50. #include "relacy/relacy_std.hpp"
  51. #include "relacy_shims.h"
  52. // We only use malloc/free anyway, and the delete macro messes up `= delete` method declarations.
  53. // We'll override the default trait malloc ourselves without a macro.
  54. #undef new
  55. #undef delete
  56. #undef malloc
  57. #undef free
  58. #else
  59. #include <atomic> // Requires C++11. Sorry VS2010.
  60. #include <cassert>
  61. #endif
  62. #include <cstddef> // for max_align_t
  63. #include <cstdint>
  64. #include <cstdlib>
  65. #include <type_traits>
  66. #include <algorithm>
  67. #include <utility>
  68. #include <limits>
  69. #include <climits> // for CHAR_BIT
  70. #include <array>
  71. #include <thread> // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading
  72. // Platform-specific definitions of a numeric thread ID type and an invalid value
  73. namespace moodycamel {
  74. namespace details {
  75. template<typename thread_id_t>
  76. struct thread_id_converter {
  77. typedef thread_id_t thread_id_numeric_size_t;
  78. typedef thread_id_t thread_id_hash_t;
  79. static thread_id_hash_t prehash(thread_id_t const &x) { return x; }
  80. };
  81. }
  82. }
  83. #if defined(MCDBGQ_USE_RELACY)
  84. namespace moodycamel { namespace details {
  85. typedef std::uint32_t thread_id_t;
  86. static const thread_id_t invalid_thread_id = 0xFFFFFFFFU;
  87. static const thread_id_t invalid_thread_id2 = 0xFFFFFFFEU;
  88. static inline thread_id_t thread_id() { return rl::thread_index(); }
  89. } }
  90. #elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__)
  91. // No sense pulling in windows.h in a header, we'll manually declare the function
  92. // we use and rely on backwards-compatibility for this not to break
  93. extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(void);
  94. namespace moodycamel { namespace details {
  95. static_assert(sizeof(unsigned long) == sizeof(std::uint32_t), "Expected size of unsigned long to be 32 bits on Windows");
  96. typedef std::uint32_t thread_id_t;
  97. static const thread_id_t invalid_thread_id = 0; // See http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx
  98. static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU; // Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4.
  99. static inline thread_id_t thread_id() { return static_cast<thread_id_t>(::GetCurrentThreadId()); }
  100. } }
  101. #elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(MOODYCAMEL_NO_THREAD_LOCAL)
  102. namespace moodycamel { namespace details {
  103. static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes");
  104. typedef std::thread::id thread_id_t;
  105. static const thread_id_t invalid_thread_id; // Default ctor creates invalid ID
  106. // Note we don't define a invalid_thread_id2 since std::thread::id doesn't have one; it's
  107. // only used if MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is defined anyway, which it won't
  108. // be.
  109. static inline thread_id_t thread_id() { return std::this_thread::get_id(); }
  110. template<std::size_t> struct thread_id_size { };
  111. template<> struct thread_id_size<4> { typedef std::uint32_t numeric_t; };
  112. template<> struct thread_id_size<8> { typedef std::uint64_t numeric_t; };
  113. template<> struct thread_id_converter<thread_id_t> {
  114. typedef thread_id_size<sizeof(thread_id_t)>::numeric_t thread_id_numeric_size_t;
  115. #ifndef __APPLE__
  116. typedef std::size_t thread_id_hash_t;
  117. #else
  118. typedef thread_id_numeric_size_t thread_id_hash_t;
  119. #endif
  120. static thread_id_hash_t prehash(thread_id_t const& x)
  121. {
  122. #ifndef __APPLE__
  123. return std::hash<std::thread::id>()(x);
  124. #else
  125. return *reinterpret_cast<thread_id_hash_t const*>(&x);
  126. #endif
  127. }
  128. };
  129. } }
  130. #else
  131. // Use a nice trick from this answer: http://stackoverflow.com/a/8438730/21475
  132. // In order to get a numeric thread ID in a platform-independent way, we use a thread-local
  133. // static variable's address as a thread identifier :-)
  134. #if defined(__GNUC__) || defined(__INTEL_COMPILER)
  135. #define MOODYCAMEL_THREADLOCAL __thread
  136. #elif defined(_MSC_VER)
  137. #define MOODYCAMEL_THREADLOCAL __declspec(thread)
  138. #else
  139. // Assume C++11 compliant compiler
  140. #define MOODYCAMEL_THREADLOCAL thread_local
  141. #endif
  142. namespace moodycamel {
  143. namespace details {
  144. typedef std::uintptr_t thread_id_t;
  145. static const thread_id_t invalid_thread_id = 0; // Address can't be nullptr
  146. static const thread_id_t invalid_thread_id2 = 1; // Member accesses off a null pointer are also generally invalid. Plus it's not aligned.
  147. inline thread_id_t thread_id() {
  148. static MOODYCAMEL_THREADLOCAL int x;
  149. return reinterpret_cast<thread_id_t>(&x);
  150. }
  151. }
  152. }
  153. #endif
  154. // Constexpr if
  155. #ifndef MOODYCAMEL_CONSTEXPR_IF
  156. #if (defined(_MSC_VER) && defined(_HAS_CXX17) && _HAS_CXX17) || __cplusplus > 201402L
  157. #define MOODYCAMEL_CONSTEXPR_IF if constexpr
  158. #define MOODYCAMEL_MAYBE_UNUSED [[maybe_unused]]
  159. #else
  160. #define MOODYCAMEL_CONSTEXPR_IF if
  161. #define MOODYCAMEL_MAYBE_UNUSED
  162. #endif
  163. #endif
  164. // Exceptions
  165. #ifndef MOODYCAMEL_EXCEPTIONS_ENABLED
  166. #if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__))
  167. #define MOODYCAMEL_EXCEPTIONS_ENABLED
  168. #endif
  169. #endif
  170. #ifdef MOODYCAMEL_EXCEPTIONS_ENABLED
  171. #define MOODYCAMEL_TRY try
  172. #define MOODYCAMEL_CATCH(...) catch(__VA_ARGS__)
  173. #define MOODYCAMEL_RETHROW throw
  174. #define MOODYCAMEL_THROW(expr) throw (expr)
  175. #else
  176. #define MOODYCAMEL_TRY MOODYCAMEL_CONSTEXPR_IF (true)
  177. #define MOODYCAMEL_CATCH(...) else MOODYCAMEL_CONSTEXPR_IF (false)
  178. #define MOODYCAMEL_RETHROW
  179. #define MOODYCAMEL_THROW(expr)
  180. #endif
  181. #ifndef MOODYCAMEL_NOEXCEPT
  182. #if !defined(MOODYCAMEL_EXCEPTIONS_ENABLED)
  183. #define MOODYCAMEL_NOEXCEPT
  184. #define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) true
  185. #define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) true
  186. #elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1800
  187. // VS2012's std::is_nothrow_[move_]constructible is broken and returns true when it shouldn't :-(
  188. // We have to assume *all* non-trivial constructors may throw on VS2012!
  189. #define MOODYCAMEL_NOEXCEPT _NOEXCEPT
  190. #define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference<valueType>::value && std::is_move_constructible<type>::value ? std::is_trivially_move_constructible<type>::value : std::is_trivially_copy_constructible<type>::value)
  191. #define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference<valueType>::value && std::is_move_assignable<type>::value ? std::is_trivially_move_assignable<type>::value || std::is_nothrow_move_assignable<type>::value : std::is_trivially_copy_assignable<type>::value || std::is_nothrow_copy_assignable<type>::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr))
  192. #elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1900
  193. #define MOODYCAMEL_NOEXCEPT _NOEXCEPT
  194. #define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference<valueType>::value && std::is_move_constructible<type>::value ? std::is_trivially_move_constructible<type>::value || std::is_nothrow_move_constructible<type>::value : std::is_trivially_copy_constructible<type>::value || std::is_nothrow_copy_constructible<type>::value)
  195. #define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference<valueType>::value && std::is_move_assignable<type>::value ? std::is_trivially_move_assignable<type>::value || std::is_nothrow_move_assignable<type>::value : std::is_trivially_copy_assignable<type>::value || std::is_nothrow_copy_assignable<type>::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr))
  196. #else
  197. #define MOODYCAMEL_NOEXCEPT noexcept
  198. #define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) noexcept(expr)
  199. #define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) noexcept(expr)
  200. #endif
  201. #endif
  202. #ifndef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  203. #ifdef MCDBGQ_USE_RELACY
  204. #define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  205. #else
  206. // VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445
  207. // g++ <=4.7 doesn't support thread_local either.
  208. // Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work
  209. #if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__)
  210. // Assume `thread_local` is fully supported in all other C++11 compilers/platforms
  211. //#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // always disabled for now since several users report having problems with it on
  212. #endif
  213. #endif
  214. #endif
  215. // VS2012 doesn't support deleted functions.
  216. // In this case, we declare the function normally but don't define it. A link error will be generated if the function is called.
  217. #ifndef MOODYCAMEL_DELETE_FUNCTION
  218. #if defined(_MSC_VER) && _MSC_VER < 1800
  219. #define MOODYCAMEL_DELETE_FUNCTION
  220. #else
  221. #define MOODYCAMEL_DELETE_FUNCTION = delete
  222. #endif
  223. #endif
  224. namespace moodycamel {
  225. namespace details {
  226. #ifndef MOODYCAMEL_ALIGNAS
  227. // VS2013 doesn't support alignas or alignof, and align() requires a constant literal
  228. #if defined(_MSC_VER) && _MSC_VER <= 1800
  229. #define MOODYCAMEL_ALIGNAS(alignment) __declspec(align(alignment))
  230. #define MOODYCAMEL_ALIGNOF(obj) __alignof(obj)
  231. #define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) typename details::Vs2013Aligned<std::alignment_of<obj>::value, T>::type
  232. template<int Align, typename T> struct Vs2013Aligned { }; // default, unsupported alignment
  233. template<typename T> struct Vs2013Aligned<1, T> { typedef __declspec(align(1)) T type; };
  234. template<typename T> struct Vs2013Aligned<2, T> { typedef __declspec(align(2)) T type; };
  235. template<typename T> struct Vs2013Aligned<4, T> { typedef __declspec(align(4)) T type; };
  236. template<typename T> struct Vs2013Aligned<8, T> { typedef __declspec(align(8)) T type; };
  237. template<typename T> struct Vs2013Aligned<16, T> { typedef __declspec(align(16)) T type; };
  238. template<typename T> struct Vs2013Aligned<32, T> { typedef __declspec(align(32)) T type; };
  239. template<typename T> struct Vs2013Aligned<64, T> { typedef __declspec(align(64)) T type; };
  240. template<typename T> struct Vs2013Aligned<128, T> { typedef __declspec(align(128)) T type; };
  241. template<typename T> struct Vs2013Aligned<256, T> { typedef __declspec(align(256)) T type; };
  242. #else
  243. template<typename T>
  244. struct identity {
  245. typedef T type;
  246. };
  247. #define MOODYCAMEL_ALIGNAS(alignment) alignas(alignment)
  248. #define MOODYCAMEL_ALIGNOF(obj) alignof(obj)
  249. #define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) alignas(alignof(obj)) typename details::identity<T>::type
  250. #endif
  251. #endif
  252. }
  253. }
  254. // TSAN can false report races in lock-free code. To enable TSAN to be used from projects that use this one,
  255. // we can apply per-function compile-time suppression.
  256. // See https://clang.llvm.org/docs/ThreadSanitizer.html#has-feature-thread-sanitizer
  257. #define MOODYCAMEL_NO_TSAN
  258. #if defined(__has_feature)
  259. #if __has_feature(thread_sanitizer)
  260. #undef MOODYCAMEL_NO_TSAN
  261. #define MOODYCAMEL_NO_TSAN __attribute__((no_sanitize("thread")))
  262. #endif // TSAN
  263. #endif // TSAN
  264. // Compiler-specific likely/unlikely hints
  265. namespace moodycamel {
  266. namespace details {
  267. #if defined(__GNUC__)
  268. static inline bool (likely)(bool x) { return __builtin_expect((x), true); }
  269. static inline bool (unlikely)(bool x) { return __builtin_expect((x), false); }
  270. #else
  271. static inline bool (likely)(bool x) { return x; }
  272. static inline bool (unlikely)(bool x) { return x; }
  273. #endif
  274. }
  275. }
  276. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  277. #include "internal/concurrentqueue_internal_debug.h"
  278. #endif
  279. namespace moodycamel {
  280. namespace details {
  281. template<typename T>
  282. struct const_numeric_max {
  283. static_assert(std::is_integral<T>::value, "const_numeric_max can only be used with integers");
  284. static const T value = std::numeric_limits<T>::is_signed
  285. ? (static_cast<T>(1) << (sizeof(T) * CHAR_BIT - 1)) - static_cast<T>(1)
  286. : static_cast<T>(-1);
  287. };
  288. #if defined(__GLIBCXX__)
  289. typedef ::max_align_t std_max_align_t; // libstdc++ forgot to add it to std:: for a while
  290. #else
  291. typedef std::max_align_t std_max_align_t; // Others (e.g. MSVC) insist it can *only* be accessed via std::
  292. #endif
  293. // Some platforms have incorrectly set max_align_t to a type with <8 bytes alignment even while supporting
  294. // 8-byte aligned scalar values (*cough* 32-bit iOS). Work around this with our own union. See issue #64.
  295. typedef union {
  296. std_max_align_t x;
  297. long long y;
  298. void *z;
  299. } max_align_t;
  300. }
  301. // Default traits for the ConcurrentQueue. To change some of the
  302. // traits without re-implementing all of them, inherit from this
  303. // struct and shadow the declarations you wish to be different;
  304. // since the traits are used as a template type parameter, the
  305. // shadowed declarations will be used where defined, and the defaults
  306. // otherwise.
  307. struct ConcurrentQueueDefaultTraits {
  308. // General-purpose size type. std::size_t is strongly recommended.
  309. typedef std::size_t size_t;
  310. // The type used for the enqueue and dequeue indices. Must be at least as
  311. // large as size_t. Should be significantly larger than the number of elements
  312. // you expect to hold at once, especially if you have a high turnover rate;
  313. // for example, on 32-bit x86, if you expect to have over a hundred million
  314. // elements or pump several million elements through your queue in a very
  315. // short space of time, using a 32-bit type *may* trigger a race condition.
  316. // A 64-bit int type is recommended in that case, and in practice will
  317. // prevent a race condition no matter the usage of the queue. Note that
  318. // whether the queue is lock-free with a 64-int type depends on the whether
  319. // std::atomic<std::uint64_t> is lock-free, which is platform-specific.
  320. typedef std::size_t index_t;
  321. // Internally, all elements are enqueued and dequeued from multi-element
  322. // blocks; this is the smallest controllable unit. If you expect few elements
  323. // but many producers, a smaller block size should be favoured. For few producers
  324. // and/or many elements, a larger block size is preferred. A sane default
  325. // is provided. Must be a power of 2.
  326. static const size_t BLOCK_SIZE = 32;
  327. // For explicit producers (i.e. when using a producer token), the block is
  328. // checked for being empty by iterating through a list of flags, one per element.
  329. // For large block sizes, this is too inefficient, and switching to an atomic
  330. // counter-based approach is faster. The switch is made for block sizes strictly
  331. // larger than this threshold.
  332. static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = 32;
  333. // How many full blocks can be expected for a single explicit producer? This should
  334. // reflect that number's maximum for optimal performance. Must be a power of 2.
  335. static const size_t EXPLICIT_INITIAL_INDEX_SIZE = 32;
  336. // How many full blocks can be expected for a single implicit producer? This should
  337. // reflect that number's maximum for optimal performance. Must be a power of 2.
  338. static const size_t IMPLICIT_INITIAL_INDEX_SIZE = 32;
  339. // The initial size of the hash table mapping thread IDs to implicit producers.
  340. // Note that the hash is resized every time it becomes half full.
  341. // Must be a power of two, and either 0 or at least 1. If 0, implicit production
  342. // (using the enqueue methods without an explicit producer token) is disabled.
  343. static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = 32;
  344. // Controls the number of items that an explicit consumer (i.e. one with a token)
  345. // must consume before it causes all consumers to rotate and move on to the next
  346. // internal queue.
  347. static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = 256;
  348. // The maximum number of elements (inclusive) that can be enqueued to a sub-queue.
  349. // Enqueue operations that would cause this limit to be surpassed will fail. Note
  350. // that this limit is enforced at the block level (for performance reasons), i.e.
  351. // it's rounded up to the nearest block size.
  352. static const size_t MAX_SUBQUEUE_SIZE = details::const_numeric_max<size_t>::value;
  353. // The number of times to spin before sleeping when waiting on a semaphore.
  354. // Recommended values are on the order of 1000-10000 unless the number of
  355. // consumer threads exceeds the number of idle cores (in which case try 0-100).
  356. // Only affects instances of the BlockingConcurrentQueue.
  357. static const int MAX_SEMA_SPINS = 10000;
  358. #ifndef MCDBGQ_USE_RELACY
  359. // Memory allocation can be customized if needed.
  360. // malloc should return nullptr on failure, and handle alignment like std::malloc.
  361. #if defined(malloc) || defined(free)
  362. // Gah, this is 2015, stop defining macros that break standard code already!
  363. // Work around malloc/free being special macros:
  364. static inline void* WORKAROUND_malloc(size_t size) { return malloc(size); }
  365. static inline void WORKAROUND_free(void* ptr) { return free(ptr); }
  366. static inline void* (malloc)(size_t size) { return WORKAROUND_malloc(size); }
  367. static inline void (free)(void* ptr) { return WORKAROUND_free(ptr); }
  368. #else
  369. static inline void *malloc(size_t size) { return std::malloc(size); }
  370. static inline void free(void *ptr) { return std::free(ptr); }
  371. #endif
  372. #else
  373. // Debug versions when running under the Relacy race detector (ignore
  374. // these in user code)
  375. static inline void* malloc(size_t size) { return rl::rl_malloc(size, $); }
  376. static inline void free(void* ptr) { return rl::rl_free(ptr, $); }
  377. #endif
  378. };
  379. // When producing or consuming many elements, the most efficient way is to:
  380. // 1) Use one of the bulk-operation methods of the queue with a token
  381. // 2) Failing that, use the bulk-operation methods without a token
  382. // 3) Failing that, create a token and use that with the single-item methods
  383. // 4) Failing that, use the single-parameter methods of the queue
  384. // Having said that, don't create tokens willy-nilly -- ideally there should be
  385. // a maximum of one token per thread (of each kind).
  386. struct ProducerToken;
  387. struct ConsumerToken;
  388. template<typename T, typename Traits>
  389. class ConcurrentQueue;
  390. template<typename T, typename Traits>
  391. class BlockingConcurrentQueue;
  392. class ConcurrentQueueTests;
  393. namespace details {
  394. struct ConcurrentQueueProducerTypelessBase {
  395. ConcurrentQueueProducerTypelessBase *next;
  396. std::atomic<bool> inactive;
  397. ProducerToken *token;
  398. ConcurrentQueueProducerTypelessBase()
  399. : next(nullptr), inactive(false), token(nullptr) {
  400. }
  401. };
  402. template<bool use32>
  403. struct _hash_32_or_64 {
  404. static inline std::uint32_t hash(std::uint32_t h) {
  405. // MurmurHash3 finalizer -- see https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
  406. // Since the thread ID is already unique, all we really want to do is propagate that
  407. // uniqueness evenly across all the bits, so that we can use a subset of the bits while
  408. // reducing collisions significantly
  409. h ^= h >> 16;
  410. h *= 0x85ebca6b;
  411. h ^= h >> 13;
  412. h *= 0xc2b2ae35;
  413. return h ^ (h >> 16);
  414. }
  415. };
  416. template<>
  417. struct _hash_32_or_64<1> {
  418. static inline std::uint64_t hash(std::uint64_t h) {
  419. h ^= h >> 33;
  420. h *= 0xff51afd7ed558ccd;
  421. h ^= h >> 33;
  422. h *= 0xc4ceb9fe1a85ec53;
  423. return h ^ (h >> 33);
  424. }
  425. };
  426. template<std::size_t size>
  427. struct hash_32_or_64 : public _hash_32_or_64<(size > 4)> {
  428. };
  429. static inline size_t hash_thread_id(thread_id_t id) {
  430. static_assert(sizeof(thread_id_t) <= 8, "Expected a platform where thread IDs are at most 64-bit values");
  431. return static_cast<size_t>(hash_32_or_64<sizeof(thread_id_converter<thread_id_t>::thread_id_hash_t)>::hash(
  432. thread_id_converter<thread_id_t>::prehash(id)));
  433. }
  434. template<typename T>
  435. static inline bool circular_less_than(T a, T b) {
  436. #ifdef _MSC_VER
  437. #pragma warning(push)
  438. #pragma warning(disable: 4554)
  439. #endif
  440. static_assert(std::is_integral<T>::value && !std::numeric_limits<T>::is_signed,
  441. "circular_less_than is intended to be used only with unsigned integer types");
  442. return static_cast<T>(a - b) >
  443. static_cast<T>(static_cast<T>(1) << static_cast<T>(sizeof(T) * CHAR_BIT - 1));
  444. #ifdef _MSC_VER
  445. #pragma warning(pop)
  446. #endif
  447. }
  448. template<typename U>
  449. static inline char *align_for(char *ptr) {
  450. const std::size_t alignment = std::alignment_of<U>::value;
  451. return ptr + (alignment - (reinterpret_cast<std::uintptr_t>(ptr) % alignment)) % alignment;
  452. }
  453. template<typename T>
  454. static inline T ceil_to_pow_2(T x) {
  455. static_assert(std::is_integral<T>::value && !std::numeric_limits<T>::is_signed,
  456. "ceil_to_pow_2 is intended to be used only with unsigned integer types");
  457. // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
  458. --x;
  459. x |= x >> 1;
  460. x |= x >> 2;
  461. x |= x >> 4;
  462. for (std::size_t i = 1; i < sizeof(T); i <<= 1) {
  463. x |= x >> (i << 3);
  464. }
  465. ++x;
  466. return x;
  467. }
  468. template<typename T>
  469. static inline void swap_relaxed(std::atomic <T> &left, std::atomic <T> &right) {
  470. T temp = std::move(left.load(std::memory_order_relaxed));
  471. left.store(std::move(right.load(std::memory_order_relaxed)), std::memory_order_relaxed);
  472. right.store(std::move(temp), std::memory_order_relaxed);
  473. }
  474. template<typename T>
  475. static inline T const &nomove(T const &x) {
  476. return x;
  477. }
  478. template<bool Enable>
  479. struct nomove_if {
  480. template<typename T>
  481. static inline T const &eval(T const &x) {
  482. return x;
  483. }
  484. };
  485. template<>
  486. struct nomove_if<false> {
  487. template<typename U>
  488. static inline auto eval(U &&x)
  489. ->
  490. decltype (std::forward<U>(x)) {
  491. return std::forward<U>(x);
  492. }
  493. };
  494. template<typename It>
  495. static inline auto deref_noexcept(It &it)
  496. MOODYCAMEL_NOEXCEPT ->
  497. decltype(*it)
  498. {
  499. return *it;
  500. }
  501. #if defined(__clang__) || !defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)
  502. template<typename T>
  503. struct is_trivially_destructible : std::is_trivially_destructible<T> {
  504. };
  505. #else
  506. template<typename T> struct is_trivially_destructible : std::has_trivial_destructor<T> { };
  507. #endif
  508. #ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  509. #ifdef MCDBGQ_USE_RELACY
  510. typedef RelacyThreadExitListener ThreadExitListener;
  511. typedef RelacyThreadExitNotifier ThreadExitNotifier;
  512. #else
  513. struct ThreadExitListener
  514. {
  515. typedef void (*callback_t)(void*);
  516. callback_t callback;
  517. void* userData;
  518. ThreadExitListener* next; // reserved for use by the ThreadExitNotifier
  519. };
  520. class ThreadExitNotifier
  521. {
  522. public:
  523. static void subscribe(ThreadExitListener* listener)
  524. {
  525. auto& tlsInst = instance();
  526. listener->next = tlsInst.tail;
  527. tlsInst.tail = listener;
  528. }
  529. static void unsubscribe(ThreadExitListener* listener)
  530. {
  531. auto& tlsInst = instance();
  532. ThreadExitListener** prev = &tlsInst.tail;
  533. for (auto ptr = tlsInst.tail; ptr != nullptr; ptr = ptr->next) {
  534. if (ptr == listener) {
  535. *prev = ptr->next;
  536. break;
  537. }
  538. prev = &ptr->next;
  539. }
  540. }
  541. private:
  542. ThreadExitNotifier() : tail(nullptr) { }
  543. ThreadExitNotifier(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION;
  544. ThreadExitNotifier& operator=(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION;
  545. ~ThreadExitNotifier()
  546. {
  547. // This thread is about to exit, let everyone know!
  548. assert(this == &instance() && "If this assert fails, you likely have a buggy compiler! Change the preprocessor conditions such that MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is no longer defined.");
  549. for (auto ptr = tail; ptr != nullptr; ptr = ptr->next) {
  550. ptr->callback(ptr->userData);
  551. }
  552. }
  553. // Thread-local
  554. static inline ThreadExitNotifier& instance()
  555. {
  556. static thread_local ThreadExitNotifier notifier;
  557. return notifier;
  558. }
  559. private:
  560. ThreadExitListener* tail;
  561. };
  562. #endif
  563. #endif
  564. template<typename T>
  565. struct static_is_lock_free_num {
  566. enum {
  567. value = 0
  568. };
  569. };
  570. template<>
  571. struct static_is_lock_free_num<signed char> {
  572. enum {
  573. value = ATOMIC_CHAR_LOCK_FREE
  574. };
  575. };
  576. template<>
  577. struct static_is_lock_free_num<short> {
  578. enum {
  579. value = ATOMIC_SHORT_LOCK_FREE
  580. };
  581. };
  582. template<>
  583. struct static_is_lock_free_num<int> {
  584. enum {
  585. value = ATOMIC_INT_LOCK_FREE
  586. };
  587. };
  588. template<>
  589. struct static_is_lock_free_num<long> {
  590. enum {
  591. value = ATOMIC_LONG_LOCK_FREE
  592. };
  593. };
  594. template<>
  595. struct static_is_lock_free_num<long long> {
  596. enum {
  597. value = ATOMIC_LLONG_LOCK_FREE
  598. };
  599. };
  600. template<typename T>
  601. struct static_is_lock_free : static_is_lock_free_num<typename std::make_signed<T>::type> {
  602. };
  603. template<>
  604. struct static_is_lock_free<bool> {
  605. enum {
  606. value = ATOMIC_BOOL_LOCK_FREE
  607. };
  608. };
  609. template<typename U>
  610. struct static_is_lock_free<U *> {
  611. enum {
  612. value = ATOMIC_POINTER_LOCK_FREE
  613. };
  614. };
  615. }
  616. struct ProducerToken {
  617. template<typename T, typename Traits>
  618. explicit ProducerToken(ConcurrentQueue<T, Traits> &queue);
  619. template<typename T, typename Traits>
  620. explicit ProducerToken(BlockingConcurrentQueue<T, Traits> &queue);
  621. ProducerToken(ProducerToken &&other)
  622. MOODYCAMEL_NOEXCEPT
  623. : producer(other.producer)
  624. {
  625. other.producer = nullptr;
  626. if (producer != nullptr) {
  627. producer->token = this;
  628. }
  629. }
  630. inline ProducerToken &operator=(ProducerToken &&other)
  631. MOODYCAMEL_NOEXCEPT
  632. {
  633. swap(other);
  634. return *this;
  635. }
  636. void swap(ProducerToken &other)
  637. MOODYCAMEL_NOEXCEPT
  638. {
  639. std::swap(producer, other.producer);
  640. if (producer != nullptr) {
  641. producer->token = this;
  642. }
  643. if (other.producer != nullptr) {
  644. other.producer->token = &other;
  645. }
  646. }
  647. // A token is always valid unless:
  648. // 1) Memory allocation failed during construction
  649. // 2) It was moved via the move constructor
  650. // (Note: assignment does a swap, leaving both potentially valid)
  651. // 3) The associated queue was destroyed
  652. // Note that if valid() returns true, that only indicates
  653. // that the token is valid for use with a specific queue,
  654. // but not which one; that's up to the user to track.
  655. inline bool valid() const { return producer != nullptr; }
  656. ~ProducerToken() {
  657. if (producer != nullptr) {
  658. producer->token = nullptr;
  659. producer->inactive.store(true, std::memory_order_release);
  660. }
  661. }
  662. // Disable copying and assignment
  663. ProducerToken(ProducerToken const &) MOODYCAMEL_DELETE_FUNCTION;
  664. ProducerToken &operator=(ProducerToken const &) MOODYCAMEL_DELETE_FUNCTION;
  665. private:
  666. template<typename T, typename Traits> friend
  667. class ConcurrentQueue;
  668. friend class ConcurrentQueueTests;
  669. protected:
  670. details::ConcurrentQueueProducerTypelessBase *producer;
  671. };
  672. struct ConsumerToken {
  673. template<typename T, typename Traits>
  674. explicit ConsumerToken(ConcurrentQueue<T, Traits> &q);
  675. template<typename T, typename Traits>
  676. explicit ConsumerToken(BlockingConcurrentQueue<T, Traits> &q);
  677. ConsumerToken(ConsumerToken &&other)
  678. MOODYCAMEL_NOEXCEPT
  679. : initialOffset(other.initialOffset), lastKnownGlobalOffset(other
  680. .lastKnownGlobalOffset),
  681. itemsConsumedFromCurrent(other
  682. .itemsConsumedFromCurrent),
  683. currentProducer(other
  684. .currentProducer),
  685. desiredProducer(other
  686. .desiredProducer)
  687. {
  688. }
  689. inline ConsumerToken &operator=(ConsumerToken &&other)
  690. MOODYCAMEL_NOEXCEPT
  691. {
  692. swap(other);
  693. return *this;
  694. }
  695. void swap(ConsumerToken &other)
  696. MOODYCAMEL_NOEXCEPT
  697. {
  698. std::swap(initialOffset, other.initialOffset);
  699. std::swap(lastKnownGlobalOffset, other.lastKnownGlobalOffset);
  700. std::swap(itemsConsumedFromCurrent, other.itemsConsumedFromCurrent);
  701. std::swap(currentProducer, other.currentProducer);
  702. std::swap(desiredProducer, other.desiredProducer);
  703. }
  704. // Disable copying and assignment
  705. ConsumerToken(ConsumerToken const &) MOODYCAMEL_DELETE_FUNCTION;
  706. ConsumerToken &operator=(ConsumerToken const &) MOODYCAMEL_DELETE_FUNCTION;
  707. private:
  708. template<typename T, typename Traits> friend
  709. class ConcurrentQueue;
  710. friend class ConcurrentQueueTests;
  711. private: // but shared with ConcurrentQueue
  712. std::uint32_t initialOffset;
  713. std::uint32_t lastKnownGlobalOffset;
  714. std::uint32_t itemsConsumedFromCurrent;
  715. details::ConcurrentQueueProducerTypelessBase *currentProducer;
  716. details::ConcurrentQueueProducerTypelessBase *desiredProducer;
  717. };
  718. // Need to forward-declare this swap because it's in a namespace.
  719. // See http://stackoverflow.com/questions/4492062/why-does-a-c-friend-class-need-a-forward-declaration-only-in-other-namespaces
  720. template<typename T, typename Traits>
  721. inline void swap(typename ConcurrentQueue<T, Traits>::ImplicitProducerKVP &a,
  722. typename ConcurrentQueue<T, Traits>::ImplicitProducerKVP &b)
  723. MOODYCAMEL_NOEXCEPT;
  724. template<typename T, typename Traits = ConcurrentQueueDefaultTraits>
  725. class ConcurrentQueue {
  726. public:
  727. typedef ::moodycamel::ProducerToken producer_token_t;
  728. typedef ::moodycamel::ConsumerToken consumer_token_t;
  729. typedef typename Traits::index_t index_t;
  730. typedef typename Traits::size_t size_t;
  731. static const size_t BLOCK_SIZE = static_cast<size_t>(Traits::BLOCK_SIZE);
  732. static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = static_cast<size_t>(Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD);
  733. static const size_t EXPLICIT_INITIAL_INDEX_SIZE = static_cast<size_t>(Traits::EXPLICIT_INITIAL_INDEX_SIZE);
  734. static const size_t IMPLICIT_INITIAL_INDEX_SIZE = static_cast<size_t>(Traits::IMPLICIT_INITIAL_INDEX_SIZE);
  735. static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = static_cast<size_t>(Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE);
  736. static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = static_cast<std::uint32_t>(Traits::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE);
  737. #ifdef _MSC_VER
  738. #pragma warning(push)
  739. #pragma warning(disable: 4307) // + integral constant overflow (that's what the ternary expression is for!)
  740. #pragma warning(disable: 4309) // static_cast: Truncation of constant value
  741. #endif
  742. static const size_t MAX_SUBQUEUE_SIZE = (details::const_numeric_max<size_t>::value -
  743. static_cast<size_t>(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE)
  744. ? details::const_numeric_max<size_t>::value : (
  745. (static_cast<size_t>(Traits::MAX_SUBQUEUE_SIZE) +
  746. (BLOCK_SIZE - 1)) / BLOCK_SIZE * BLOCK_SIZE);
  747. #ifdef _MSC_VER
  748. #pragma warning(pop)
  749. #endif
  750. static_assert(!
  751. std::numeric_limits<size_t>::is_signed &&std::is_integral<size_t>::value,
  752. "Traits::size_t must be an unsigned integral type");
  753. static_assert(!
  754. std::numeric_limits<index_t>::is_signed &&std::is_integral<index_t>::value,
  755. "Traits::index_t must be an unsigned integral type");
  756. static_assert(sizeof(index_t) >= sizeof(size_t), "Traits::index_t must be at least as wide as Traits::size_t");
  757. static_assert((BLOCK_SIZE
  758. > 1) && !(
  759. BLOCK_SIZE &(BLOCK_SIZE
  760. - 1)), "Traits::BLOCK_SIZE must be a power of 2 (and at least 2)");
  761. static_assert((EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD
  762. > 1) && !(
  763. EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD &(EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD
  764. - 1)), "Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD must be a power of 2 (and greater than 1)");
  765. static_assert((EXPLICIT_INITIAL_INDEX_SIZE
  766. > 1) && !(
  767. EXPLICIT_INITIAL_INDEX_SIZE &(EXPLICIT_INITIAL_INDEX_SIZE
  768. - 1)), "Traits::EXPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)");
  769. static_assert((IMPLICIT_INITIAL_INDEX_SIZE
  770. > 1) && !(
  771. IMPLICIT_INITIAL_INDEX_SIZE &(IMPLICIT_INITIAL_INDEX_SIZE
  772. - 1)), "Traits::IMPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)");
  773. static_assert((INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
  774. == 0) || !(
  775. INITIAL_IMPLICIT_PRODUCER_HASH_SIZE &(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
  776. - 1)), "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be a power of 2");
  777. static_assert(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
  778. == 0 || INITIAL_IMPLICIT_PRODUCER_HASH_SIZE >= 1, "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be at least 1 (or 0 to disable implicit enqueueing)");
  779. public:
  780. // Creates a queue with at least `capacity` element slots; note that the
  781. // actual number of elements that can be inserted without additional memory
  782. // allocation depends on the number of producers and the block size (e.g. if
  783. // the block size is equal to `capacity`, only a single block will be allocated
  784. // up-front, which means only a single producer will be able to enqueue elements
  785. // without an extra allocation -- blocks aren't shared between producers).
  786. // This method is not thread safe -- it is up to the user to ensure that the
  787. // queue is fully constructed before it starts being used by other threads (this
  788. // includes making the memory effects of construction visible, possibly with a
  789. // memory barrier).
  790. explicit ConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE)
  791. : producerListTail(nullptr),
  792. producerCount(0),
  793. initialBlockPoolIndex(0),
  794. nextExplicitConsumerId(0),
  795. globalExplicitConsumerOffset(0) {
  796. implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed);
  797. populate_initial_implicit_producer_hash();
  798. populate_initial_block_list(capacity / BLOCK_SIZE + ((capacity & (BLOCK_SIZE - 1)) == 0 ? 0 : 1));
  799. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  800. // Track all the producers using a fully-resolved typed list for
  801. // each kind; this makes it possible to debug them starting from
  802. // the root queue object (otherwise wacky casts are needed that
  803. // don't compile in the debugger's expression evaluator).
  804. explicitProducers.store(nullptr, std::memory_order_relaxed);
  805. implicitProducers.store(nullptr, std::memory_order_relaxed);
  806. #endif
  807. }
  808. // Computes the correct amount of pre-allocated blocks for you based
  809. // on the minimum number of elements you want available at any given
  810. // time, and the maximum concurrent number of each type of producer.
  811. ConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers)
  812. : producerListTail(nullptr),
  813. producerCount(0),
  814. initialBlockPoolIndex(0),
  815. nextExplicitConsumerId(0),
  816. globalExplicitConsumerOffset(0) {
  817. implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed);
  818. populate_initial_implicit_producer_hash();
  819. size_t blocks = (((minCapacity + BLOCK_SIZE - 1) / BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) +
  820. 2 * (maxExplicitProducers + maxImplicitProducers);
  821. populate_initial_block_list(blocks);
  822. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  823. explicitProducers.store(nullptr, std::memory_order_relaxed);
  824. implicitProducers.store(nullptr, std::memory_order_relaxed);
  825. #endif
  826. }
  827. // Note: The queue should not be accessed concurrently while it's
  828. // being deleted. It's up to the user to synchronize this.
  829. // This method is not thread safe.
  830. ~ConcurrentQueue() {
  831. // Destroy producers
  832. auto ptr = producerListTail.load(std::memory_order_relaxed);
  833. while (ptr != nullptr) {
  834. auto next = ptr->next_prod();
  835. if (ptr->token != nullptr) {
  836. ptr->token->producer = nullptr;
  837. }
  838. destroy(ptr);
  839. ptr = next;
  840. }
  841. // Destroy implicit producer hash tables
  842. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE != 0) {
  843. auto hash = implicitProducerHash.load(std::memory_order_relaxed);
  844. while (hash != nullptr) {
  845. auto prev = hash->prev;
  846. if (prev !=
  847. nullptr) { // The last hash is part of this object and was not allocated dynamically
  848. for (size_t i = 0; i != hash->capacity; ++i) {
  849. hash->entries[i].~ImplicitProducerKVP();
  850. }
  851. hash->~ImplicitProducerHash();
  852. (Traits::free)(hash);
  853. }
  854. hash = prev;
  855. }
  856. }
  857. // Destroy global free list
  858. auto block = freeList.head_unsafe();
  859. while (block != nullptr) {
  860. auto next = block->freeListNext.load(std::memory_order_relaxed);
  861. if (block->dynamicallyAllocated) {
  862. destroy(block);
  863. }
  864. block = next;
  865. }
  866. // Destroy initial free list
  867. destroy_array(initialBlockPool, initialBlockPoolSize);
  868. }
  869. // Disable copying and copy assignment
  870. ConcurrentQueue(ConcurrentQueue const &) MOODYCAMEL_DELETE_FUNCTION;
  871. ConcurrentQueue &operator=(ConcurrentQueue const &) MOODYCAMEL_DELETE_FUNCTION;
  872. // Moving is supported, but note that it is *not* a thread-safe operation.
  873. // Nobody can use the queue while it's being moved, and the memory effects
  874. // of that move must be propagated to other threads before they can use it.
  875. // Note: When a queue is moved, its tokens are still valid but can only be
  876. // used with the destination queue (i.e. semantically they are moved along
  877. // with the queue itself).
  878. ConcurrentQueue(ConcurrentQueue &&other)
  879. MOODYCAMEL_NOEXCEPT
  880. : producerListTail(other.producerListTail.load(std::memory_order_relaxed)),
  881. producerCount(other
  882. .producerCount.
  883. load(std::memory_order_relaxed)
  884. ),
  885. initialBlockPoolIndex(other
  886. .initialBlockPoolIndex.
  887. load(std::memory_order_relaxed)
  888. ),
  889. initialBlockPool(other
  890. .initialBlockPool),
  891. initialBlockPoolSize(other
  892. .initialBlockPoolSize),
  893. freeList (std::move(other
  894. .freeList)),
  895. nextExplicitConsumerId(other
  896. .nextExplicitConsumerId.
  897. load(std::memory_order_relaxed)
  898. ),
  899. globalExplicitConsumerOffset(other
  900. .globalExplicitConsumerOffset.
  901. load(std::memory_order_relaxed)
  902. )
  903. {
  904. // Move the other one into this, and leave the other one as an empty queue
  905. implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed);
  906. populate_initial_implicit_producer_hash();
  907. swap_implicit_producer_hashes(other);
  908. other.producerListTail.store(nullptr, std::memory_order_relaxed);
  909. other.producerCount.store(0, std::memory_order_relaxed);
  910. other.nextExplicitConsumerId.store(0, std::memory_order_relaxed);
  911. other.globalExplicitConsumerOffset.store(0, std::memory_order_relaxed);
  912. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  913. explicitProducers.store(other.explicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed);
  914. other.explicitProducers.store(nullptr, std::memory_order_relaxed);
  915. implicitProducers.store(other.implicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed);
  916. other.implicitProducers.store(nullptr, std::memory_order_relaxed);
  917. #endif
  918. other.initialBlockPoolIndex.store(0, std::memory_order_relaxed);
  919. other.initialBlockPoolSize = 0;
  920. other.initialBlockPool = nullptr;
  921. reown_producers();
  922. }
  923. inline ConcurrentQueue &operator=(ConcurrentQueue &&other)
  924. MOODYCAMEL_NOEXCEPT
  925. {
  926. return swap_internal(other);
  927. }
  928. // Swaps this queue's state with the other's. Not thread-safe.
  929. // Swapping two queues does not invalidate their tokens, however
  930. // the tokens that were created for one queue must be used with
  931. // only the swapped queue (i.e. the tokens are tied to the
  932. // queue's movable state, not the object itself).
  933. inline void swap(ConcurrentQueue &other)
  934. MOODYCAMEL_NOEXCEPT
  935. {
  936. swap_internal(other);
  937. }
  938. private:
  939. ConcurrentQueue &swap_internal(ConcurrentQueue &other) {
  940. if (this == &other) {
  941. return *this;
  942. }
  943. details::swap_relaxed(producerListTail, other.producerListTail);
  944. details::swap_relaxed(producerCount, other.producerCount);
  945. details::swap_relaxed(initialBlockPoolIndex, other.initialBlockPoolIndex);
  946. std::swap(initialBlockPool, other.initialBlockPool);
  947. std::swap(initialBlockPoolSize, other.initialBlockPoolSize);
  948. freeList.swap(other.freeList);
  949. details::swap_relaxed(nextExplicitConsumerId, other.nextExplicitConsumerId);
  950. details::swap_relaxed(globalExplicitConsumerOffset, other.globalExplicitConsumerOffset);
  951. swap_implicit_producer_hashes(other);
  952. reown_producers();
  953. other.reown_producers();
  954. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  955. details::swap_relaxed(explicitProducers, other.explicitProducers);
  956. details::swap_relaxed(implicitProducers, other.implicitProducers);
  957. #endif
  958. return *this;
  959. }
  960. public:
  961. // Enqueues a single item (by copying it).
  962. // Allocates memory if required. Only fails if memory allocation fails (or implicit
  963. // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0,
  964. // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
  965. // Thread-safe.
  966. inline bool enqueue(T const &item) {
  967. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false;
  968. else return inner_enqueue<CanAlloc>(item);
  969. }
  970. // Enqueues a single item (by moving it, if possible).
  971. // Allocates memory if required. Only fails if memory allocation fails (or implicit
  972. // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0,
  973. // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
  974. // Thread-safe.
  975. inline bool enqueue(T &&item) {
  976. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false;
  977. else return inner_enqueue<CanAlloc>(std::move(item));
  978. }
  979. // Enqueues a single item (by copying it) using an explicit producer token.
  980. // Allocates memory if required. Only fails if memory allocation fails (or
  981. // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
  982. // Thread-safe.
  983. inline bool enqueue(producer_token_t const &token, T const &item) {
  984. return inner_enqueue<CanAlloc>(token, item);
  985. }
  986. // Enqueues a single item (by moving it, if possible) using an explicit producer token.
  987. // Allocates memory if required. Only fails if memory allocation fails (or
  988. // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
  989. // Thread-safe.
  990. inline bool enqueue(producer_token_t const &token, T &&item) {
  991. return inner_enqueue<CanAlloc>(token, std::move(item));
  992. }
  993. // Enqueues several items.
  994. // Allocates memory if required. Only fails if memory allocation fails (or
  995. // implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
  996. // is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
  997. // Note: Use std::make_move_iterator if the elements should be moved instead of copied.
  998. // Thread-safe.
  999. template<typename It>
  1000. bool enqueue_bulk(It itemFirst, size_t count) {
  1001. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false;
  1002. else return inner_enqueue_bulk<CanAlloc>(itemFirst, count);
  1003. }
  1004. // Enqueues several items using an explicit producer token.
  1005. // Allocates memory if required. Only fails if memory allocation fails
  1006. // (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
  1007. // Note: Use std::make_move_iterator if the elements should be moved
  1008. // instead of copied.
  1009. // Thread-safe.
  1010. template<typename It>
  1011. bool enqueue_bulk(producer_token_t const &token, It itemFirst, size_t count) {
  1012. return inner_enqueue_bulk<CanAlloc>(token, itemFirst, count);
  1013. }
  1014. // Enqueues a single item (by copying it).
  1015. // Does not allocate memory. Fails if not enough room to enqueue (or implicit
  1016. // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
  1017. // is 0).
  1018. // Thread-safe.
  1019. inline bool try_enqueue(T const &item) {
  1020. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false;
  1021. else return inner_enqueue<CannotAlloc>(item);
  1022. }
  1023. // Enqueues a single item (by moving it, if possible).
  1024. // Does not allocate memory (except for one-time implicit producer).
  1025. // Fails if not enough room to enqueue (or implicit production is
  1026. // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0).
  1027. // Thread-safe.
  1028. inline bool try_enqueue(T &&item) {
  1029. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false;
  1030. else return inner_enqueue<CannotAlloc>(std::move(item));
  1031. }
  1032. // Enqueues a single item (by copying it) using an explicit producer token.
  1033. // Does not allocate memory. Fails if not enough room to enqueue.
  1034. // Thread-safe.
  1035. inline bool try_enqueue(producer_token_t const &token, T const &item) {
  1036. return inner_enqueue<CannotAlloc>(token, item);
  1037. }
  1038. // Enqueues a single item (by moving it, if possible) using an explicit producer token.
  1039. // Does not allocate memory. Fails if not enough room to enqueue.
  1040. // Thread-safe.
  1041. inline bool try_enqueue(producer_token_t const &token, T &&item) {
  1042. return inner_enqueue<CannotAlloc>(token, std::move(item));
  1043. }
  1044. // Enqueues several items.
  1045. // Does not allocate memory (except for one-time implicit producer).
  1046. // Fails if not enough room to enqueue (or implicit production is
  1047. // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0).
  1048. // Note: Use std::make_move_iterator if the elements should be moved
  1049. // instead of copied.
  1050. // Thread-safe.
  1051. template<typename It>
  1052. bool try_enqueue_bulk(It itemFirst, size_t count) {
  1053. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false;
  1054. else return inner_enqueue_bulk<CannotAlloc>(itemFirst, count);
  1055. }
  1056. // Enqueues several items using an explicit producer token.
  1057. // Does not allocate memory. Fails if not enough room to enqueue.
  1058. // Note: Use std::make_move_iterator if the elements should be moved
  1059. // instead of copied.
  1060. // Thread-safe.
  1061. template<typename It>
  1062. bool try_enqueue_bulk(producer_token_t const &token, It itemFirst, size_t count) {
  1063. return inner_enqueue_bulk<CannotAlloc>(token, itemFirst, count);
  1064. }
  1065. // Attempts to dequeue from the queue.
  1066. // Returns false if all producer streams appeared empty at the time they
  1067. // were checked (so, the queue is likely but not guaranteed to be empty).
  1068. // Never allocates. Thread-safe.
  1069. template<typename U>
  1070. bool try_dequeue(U &item) {
  1071. // Instead of simply trying each producer in turn (which could cause needless contention on the first
  1072. // producer), we score them heuristically.
  1073. size_t nonEmptyCount = 0;
  1074. ProducerBase *best = nullptr;
  1075. size_t bestSize = 0;
  1076. for (auto ptr = producerListTail.load(std::memory_order_acquire);
  1077. nonEmptyCount < 3 && ptr != nullptr; ptr = ptr->next_prod()) {
  1078. auto size = ptr->size_approx();
  1079. if (size > 0) {
  1080. if (size > bestSize) {
  1081. bestSize = size;
  1082. best = ptr;
  1083. }
  1084. ++nonEmptyCount;
  1085. }
  1086. }
  1087. // If there was at least one non-empty queue but it appears empty at the time
  1088. // we try to dequeue from it, we need to make sure every queue's been tried
  1089. if (nonEmptyCount > 0) {
  1090. if ((details::likely)(best->dequeue(item))) {
  1091. return true;
  1092. }
  1093. for (auto ptr = producerListTail.load(std::memory_order_acquire);
  1094. ptr != nullptr; ptr = ptr->next_prod()) {
  1095. if (ptr != best && ptr->dequeue(item)) {
  1096. return true;
  1097. }
  1098. }
  1099. }
  1100. return false;
  1101. }
  1102. // Attempts to dequeue from the queue.
  1103. // Returns false if all producer streams appeared empty at the time they
  1104. // were checked (so, the queue is likely but not guaranteed to be empty).
  1105. // This differs from the try_dequeue(item) method in that this one does
  1106. // not attempt to reduce contention by interleaving the order that producer
  1107. // streams are dequeued from. So, using this method can reduce overall throughput
  1108. // under contention, but will give more predictable results in single-threaded
  1109. // consumer scenarios. This is mostly only useful for internal unit tests.
  1110. // Never allocates. Thread-safe.
  1111. template<typename U>
  1112. bool try_dequeue_non_interleaved(U &item) {
  1113. for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) {
  1114. if (ptr->dequeue(item)) {
  1115. return true;
  1116. }
  1117. }
  1118. return false;
  1119. }
  1120. // Attempts to dequeue from the queue using an explicit consumer token.
  1121. // Returns false if all producer streams appeared empty at the time they
  1122. // were checked (so, the queue is likely but not guaranteed to be empty).
  1123. // Never allocates. Thread-safe.
  1124. template<typename U>
  1125. bool try_dequeue(consumer_token_t &token, U &item) {
  1126. // The idea is roughly as follows:
  1127. // Every 256 items from one producer, make everyone rotate (increase the global offset) -> this means the highest efficiency consumer dictates the rotation speed of everyone else, more or less
  1128. // If you see that the global offset has changed, you must reset your consumption counter and move to your designated place
  1129. // If there's no items where you're supposed to be, keep moving until you find a producer with some items
  1130. // If the global offset has not changed but you've run out of items to consume, move over from your current position until you find an producer with something in it
  1131. if (token.desiredProducer == nullptr ||
  1132. token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) {
  1133. if (!update_current_producer_after_rotation(token)) {
  1134. return false;
  1135. }
  1136. }
  1137. // If there was at least one non-empty queue but it appears empty at the time
  1138. // we try to dequeue from it, we need to make sure every queue's been tried
  1139. if (static_cast<ProducerBase *>(token.currentProducer)->dequeue(item)) {
  1140. if (++token.itemsConsumedFromCurrent == EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) {
  1141. globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed);
  1142. }
  1143. return true;
  1144. }
  1145. auto tail = producerListTail.load(std::memory_order_acquire);
  1146. auto ptr = static_cast<ProducerBase *>(token.currentProducer)->next_prod();
  1147. if (ptr == nullptr) {
  1148. ptr = tail;
  1149. }
  1150. while (ptr != static_cast<ProducerBase *>(token.currentProducer)) {
  1151. if (ptr->dequeue(item)) {
  1152. token.currentProducer = ptr;
  1153. token.itemsConsumedFromCurrent = 1;
  1154. return true;
  1155. }
  1156. ptr = ptr->next_prod();
  1157. if (ptr == nullptr) {
  1158. ptr = tail;
  1159. }
  1160. }
  1161. return false;
  1162. }
  1163. // Attempts to dequeue several elements from the queue.
  1164. // Returns the number of items actually dequeued.
  1165. // Returns 0 if all producer streams appeared empty at the time they
  1166. // were checked (so, the queue is likely but not guaranteed to be empty).
  1167. // Never allocates. Thread-safe.
  1168. template<typename It>
  1169. size_t try_dequeue_bulk(It itemFirst, size_t max) {
  1170. size_t count = 0;
  1171. for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) {
  1172. count += ptr->dequeue_bulk(itemFirst, max - count);
  1173. if (count == max) {
  1174. break;
  1175. }
  1176. }
  1177. return count;
  1178. }
  1179. // Attempts to dequeue several elements from the queue using an explicit consumer token.
  1180. // Returns the number of items actually dequeued.
  1181. // Returns 0 if all producer streams appeared empty at the time they
  1182. // were checked (so, the queue is likely but not guaranteed to be empty).
  1183. // Never allocates. Thread-safe.
  1184. template<typename It>
  1185. size_t try_dequeue_bulk(consumer_token_t &token, It itemFirst, size_t max) {
  1186. if (token.desiredProducer == nullptr ||
  1187. token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) {
  1188. if (!update_current_producer_after_rotation(token)) {
  1189. return 0;
  1190. }
  1191. }
  1192. size_t count = static_cast<ProducerBase *>(token.currentProducer)->dequeue_bulk(itemFirst, max);
  1193. if (count == max) {
  1194. if ((token.itemsConsumedFromCurrent += static_cast<std::uint32_t>(max)) >=
  1195. EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) {
  1196. globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed);
  1197. }
  1198. return max;
  1199. }
  1200. token.itemsConsumedFromCurrent += static_cast<std::uint32_t>(count);
  1201. max -= count;
  1202. auto tail = producerListTail.load(std::memory_order_acquire);
  1203. auto ptr = static_cast<ProducerBase *>(token.currentProducer)->next_prod();
  1204. if (ptr == nullptr) {
  1205. ptr = tail;
  1206. }
  1207. while (ptr != static_cast<ProducerBase *>(token.currentProducer)) {
  1208. auto dequeued = ptr->dequeue_bulk(itemFirst, max);
  1209. count += dequeued;
  1210. if (dequeued != 0) {
  1211. token.currentProducer = ptr;
  1212. token.itemsConsumedFromCurrent = static_cast<std::uint32_t>(dequeued);
  1213. }
  1214. if (dequeued == max) {
  1215. break;
  1216. }
  1217. max -= dequeued;
  1218. ptr = ptr->next_prod();
  1219. if (ptr == nullptr) {
  1220. ptr = tail;
  1221. }
  1222. }
  1223. return count;
  1224. }
  1225. // Attempts to dequeue from a specific producer's inner queue.
  1226. // If you happen to know which producer you want to dequeue from, this
  1227. // is significantly faster than using the general-case try_dequeue methods.
  1228. // Returns false if the producer's queue appeared empty at the time it
  1229. // was checked (so, the queue is likely but not guaranteed to be empty).
  1230. // Never allocates. Thread-safe.
  1231. template<typename U>
  1232. inline bool try_dequeue_from_producer(producer_token_t const &producer, U &item) {
  1233. return static_cast<ExplicitProducer *>(producer.producer)->dequeue(item);
  1234. }
  1235. // Attempts to dequeue several elements from a specific producer's inner queue.
  1236. // Returns the number of items actually dequeued.
  1237. // If you happen to know which producer you want to dequeue from, this
  1238. // is significantly faster than using the general-case try_dequeue methods.
  1239. // Returns 0 if the producer's queue appeared empty at the time it
  1240. // was checked (so, the queue is likely but not guaranteed to be empty).
  1241. // Never allocates. Thread-safe.
  1242. template<typename It>
  1243. inline size_t try_dequeue_bulk_from_producer(producer_token_t const &producer, It itemFirst, size_t max) {
  1244. return static_cast<ExplicitProducer *>(producer.producer)->dequeue_bulk(itemFirst, max);
  1245. }
  1246. // Returns an estimate of the total number of elements currently in the queue. This
  1247. // estimate is only accurate if the queue has completely stabilized before it is called
  1248. // (i.e. all enqueue and dequeue operations have completed and their memory effects are
  1249. // visible on the calling thread, and no further operations start while this method is
  1250. // being called).
  1251. // Thread-safe.
  1252. size_t size_approx() const {
  1253. size_t size = 0;
  1254. for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) {
  1255. size += ptr->size_approx();
  1256. }
  1257. return size;
  1258. }
  1259. // Returns true if the underlying atomic variables used by
  1260. // the queue are lock-free (they should be on most platforms).
  1261. // Thread-safe.
  1262. static constexpr bool is_lock_free() {
  1263. return
  1264. details::static_is_lock_free<bool>::value == 2 &&
  1265. details::static_is_lock_free<size_t>::value == 2 &&
  1266. details::static_is_lock_free<std::uint32_t>::value == 2 &&
  1267. details::static_is_lock_free<index_t>::value == 2 &&
  1268. details::static_is_lock_free<void *>::value == 2 &&
  1269. details::static_is_lock_free<typename details::thread_id_converter<details::thread_id_t>::thread_id_numeric_size_t>::value ==
  1270. 2;
  1271. }
  1272. private:
  1273. friend struct ProducerToken;
  1274. friend struct ConsumerToken;
  1275. struct ExplicitProducer;
  1276. friend struct ExplicitProducer;
  1277. struct ImplicitProducer;
  1278. friend struct ImplicitProducer;
  1279. friend class ConcurrentQueueTests;
  1280. enum AllocationMode {
  1281. CanAlloc, CannotAlloc
  1282. };
  1283. ///////////////////////////////
  1284. // Queue methods
  1285. ///////////////////////////////
  1286. template<AllocationMode canAlloc, typename U>
  1287. inline bool inner_enqueue(producer_token_t const &token, U &&element) {
  1288. return static_cast<ExplicitProducer *>(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue<canAlloc>(
  1289. std::forward<U>(element));
  1290. }
  1291. template<AllocationMode canAlloc, typename U>
  1292. inline bool inner_enqueue(U &&element) {
  1293. auto producer = get_or_add_implicit_producer();
  1294. return producer == nullptr ? false
  1295. : producer->ConcurrentQueue::ImplicitProducer::template enqueue<canAlloc>(
  1296. std::forward<U>(element));
  1297. }
  1298. template<AllocationMode canAlloc, typename It>
  1299. inline bool inner_enqueue_bulk(producer_token_t const &token, It itemFirst, size_t count) {
  1300. return static_cast<ExplicitProducer *>(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue_bulk<canAlloc>(
  1301. itemFirst, count);
  1302. }
  1303. template<AllocationMode canAlloc, typename It>
  1304. inline bool inner_enqueue_bulk(It itemFirst, size_t count) {
  1305. auto producer = get_or_add_implicit_producer();
  1306. return producer == nullptr ? false
  1307. : producer->ConcurrentQueue::ImplicitProducer::template enqueue_bulk<canAlloc>(
  1308. itemFirst, count);
  1309. }
  1310. inline bool update_current_producer_after_rotation(consumer_token_t &token) {
  1311. // Ah, there's been a rotation, figure out where we should be!
  1312. auto tail = producerListTail.load(std::memory_order_acquire);
  1313. if (token.desiredProducer == nullptr && tail == nullptr) {
  1314. return false;
  1315. }
  1316. auto prodCount = producerCount.load(std::memory_order_relaxed);
  1317. auto globalOffset = globalExplicitConsumerOffset.load(std::memory_order_relaxed);
  1318. if ((details::unlikely)(token.desiredProducer == nullptr)) {
  1319. // Aha, first time we're dequeueing anything.
  1320. // Figure out our local position
  1321. // Note: offset is from start, not end, but we're traversing from end -- subtract from count first
  1322. std::uint32_t offset = prodCount - 1 - (token.initialOffset % prodCount);
  1323. token.desiredProducer = tail;
  1324. for (std::uint32_t i = 0; i != offset; ++i) {
  1325. token.desiredProducer = static_cast<ProducerBase *>(token.desiredProducer)->next_prod();
  1326. if (token.desiredProducer == nullptr) {
  1327. token.desiredProducer = tail;
  1328. }
  1329. }
  1330. }
  1331. std::uint32_t delta = globalOffset - token.lastKnownGlobalOffset;
  1332. if (delta >= prodCount) {
  1333. delta = delta % prodCount;
  1334. }
  1335. for (std::uint32_t i = 0; i != delta; ++i) {
  1336. token.desiredProducer = static_cast<ProducerBase *>(token.desiredProducer)->next_prod();
  1337. if (token.desiredProducer == nullptr) {
  1338. token.desiredProducer = tail;
  1339. }
  1340. }
  1341. token.lastKnownGlobalOffset = globalOffset;
  1342. token.currentProducer = token.desiredProducer;
  1343. token.itemsConsumedFromCurrent = 0;
  1344. return true;
  1345. }
  1346. ///////////////////////////
  1347. // Free list
  1348. ///////////////////////////
  1349. template<typename N>
  1350. struct FreeListNode {
  1351. FreeListNode() : freeListRefs(0), freeListNext(nullptr) {}
  1352. std::atomic<std::uint32_t> freeListRefs;
  1353. std::atomic<N *> freeListNext;
  1354. };
  1355. // A simple CAS-based lock-free free list. Not the fastest thing in the world under heavy contention, but
  1356. // simple and correct (assuming nodes are never freed until after the free list is destroyed), and fairly
  1357. // speedy under low contention.
  1358. template<typename N> // N must inherit FreeListNode or have the same fields (and initialization of them)
  1359. struct FreeList {
  1360. FreeList() : freeListHead(nullptr) {}
  1361. FreeList(FreeList &&other) : freeListHead(other.freeListHead.load(std::memory_order_relaxed)) {
  1362. other.freeListHead.store(nullptr, std::memory_order_relaxed);
  1363. }
  1364. void swap(FreeList &other) { details::swap_relaxed(freeListHead, other.freeListHead); }
  1365. FreeList(FreeList const &) MOODYCAMEL_DELETE_FUNCTION;
  1366. FreeList &operator=(FreeList const &) MOODYCAMEL_DELETE_FUNCTION;
  1367. inline void add(N *node) {
  1368. #ifdef MCDBGQ_NOLOCKFREE_FREELIST
  1369. debug::DebugLock lock(mutex);
  1370. #endif
  1371. // We know that the should-be-on-freelist bit is 0 at this point, so it's safe to
  1372. // set it using a fetch_add
  1373. if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST, std::memory_order_acq_rel) == 0) {
  1374. // Oh look! We were the last ones referencing this node, and we know
  1375. // we want to add it to the free list, so let's do it!
  1376. add_knowing_refcount_is_zero(node);
  1377. }
  1378. }
  1379. inline N *try_get() {
  1380. #ifdef MCDBGQ_NOLOCKFREE_FREELIST
  1381. debug::DebugLock lock(mutex);
  1382. #endif
  1383. auto head = freeListHead.load(std::memory_order_acquire);
  1384. while (head != nullptr) {
  1385. auto prevHead = head;
  1386. auto refs = head->freeListRefs.load(std::memory_order_relaxed);
  1387. if ((refs & REFS_MASK) == 0 ||
  1388. !head->freeListRefs.compare_exchange_strong(refs, refs + 1, std::memory_order_acquire,
  1389. std::memory_order_relaxed)) {
  1390. head = freeListHead.load(std::memory_order_acquire);
  1391. continue;
  1392. }
  1393. // Good, reference count has been incremented (it wasn't at zero), which means we can read the
  1394. // next and not worry about it changing between now and the time we do the CAS
  1395. auto next = head->freeListNext.load(std::memory_order_relaxed);
  1396. if (freeListHead.compare_exchange_strong(head, next, std::memory_order_acquire,
  1397. std::memory_order_relaxed)) {
  1398. // Yay, got the node. This means it was on the list, which means shouldBeOnFreeList must be false no
  1399. // matter the refcount (because nobody else knows it's been taken off yet, it can't have been put back on).
  1400. assert((head->freeListRefs.load(std::memory_order_relaxed) & SHOULD_BE_ON_FREELIST) == 0);
  1401. // Decrease refcount twice, once for our ref, and once for the list's ref
  1402. head->freeListRefs.fetch_sub(2, std::memory_order_release);
  1403. return head;
  1404. }
  1405. // OK, the head must have changed on us, but we still need to decrease the refcount we increased.
  1406. // Note that we don't need to release any memory effects, but we do need to ensure that the reference
  1407. // count decrement happens-after the CAS on the head.
  1408. refs = prevHead->freeListRefs.fetch_sub(1, std::memory_order_acq_rel);
  1409. if (refs == SHOULD_BE_ON_FREELIST + 1) {
  1410. add_knowing_refcount_is_zero(prevHead);
  1411. }
  1412. }
  1413. return nullptr;
  1414. }
  1415. // Useful for traversing the list when there's no contention (e.g. to destroy remaining nodes)
  1416. N *head_unsafe() const { return freeListHead.load(std::memory_order_relaxed); }
  1417. private:
  1418. inline void add_knowing_refcount_is_zero(N *node) {
  1419. // Since the refcount is zero, and nobody can increase it once it's zero (except us, and we run
  1420. // only one copy of this method per node at a time, i.e. the single thread case), then we know
  1421. // we can safely change the next pointer of the node; however, once the refcount is back above
  1422. // zero, then other threads could increase it (happens under heavy contention, when the refcount
  1423. // goes to zero in between a load and a refcount increment of a node in try_get, then back up to
  1424. // something non-zero, then the refcount increment is done by the other thread) -- so, if the CAS
  1425. // to add the node to the actual list fails, decrease the refcount and leave the add operation to
  1426. // the next thread who puts the refcount back at zero (which could be us, hence the loop).
  1427. auto head = freeListHead.load(std::memory_order_relaxed);
  1428. while (true) {
  1429. node->freeListNext.store(head, std::memory_order_relaxed);
  1430. node->freeListRefs.store(1, std::memory_order_release);
  1431. if (!freeListHead.compare_exchange_strong(head, node, std::memory_order_release,
  1432. std::memory_order_relaxed)) {
  1433. // Hmm, the add failed, but we can only try again when the refcount goes back to zero
  1434. if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST - 1, std::memory_order_release) == 1) {
  1435. continue;
  1436. }
  1437. }
  1438. return;
  1439. }
  1440. }
  1441. private:
  1442. // Implemented like a stack, but where node order doesn't matter (nodes are inserted out of order under contention)
  1443. std::atomic<N *> freeListHead;
  1444. static const std::uint32_t REFS_MASK = 0x7FFFFFFF;
  1445. static const std::uint32_t SHOULD_BE_ON_FREELIST = 0x80000000;
  1446. #ifdef MCDBGQ_NOLOCKFREE_FREELIST
  1447. debug::DebugMutex mutex;
  1448. #endif
  1449. };
  1450. ///////////////////////////
  1451. // Block
  1452. ///////////////////////////
  1453. enum InnerQueueContext {
  1454. implicit_context = 0, explicit_context = 1
  1455. };
  1456. struct Block {
  1457. Block()
  1458. : next(nullptr), elementsCompletelyDequeued(0), freeListRefs(0), freeListNext(nullptr),
  1459. shouldBeOnFreeList(false), dynamicallyAllocated(true) {
  1460. #ifdef MCDBGQ_TRACKMEM
  1461. owner = nullptr;
  1462. #endif
  1463. }
  1464. template<InnerQueueContext context>
  1465. inline bool is_empty() const {
  1466. MOODYCAMEL_CONSTEXPR_IF (context == explicit_context &&
  1467. BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) {
  1468. // Check flags
  1469. for (size_t i = 0; i < BLOCK_SIZE; ++i) {
  1470. if (!emptyFlags[i].load(std::memory_order_relaxed)) {
  1471. return false;
  1472. }
  1473. }
  1474. // Aha, empty; make sure we have all other memory effects that happened before the empty flags were set
  1475. std::atomic_thread_fence(std::memory_order_acquire);
  1476. return true;
  1477. } else {
  1478. // Check counter
  1479. if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE) {
  1480. std::atomic_thread_fence(std::memory_order_acquire);
  1481. return true;
  1482. }
  1483. assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE);
  1484. return false;
  1485. }
  1486. }
  1487. // Returns true if the block is now empty (does not apply in explicit context)
  1488. template<InnerQueueContext context>
  1489. inline bool set_empty(MOODYCAMEL_MAYBE_UNUSED index_t i) {
  1490. MOODYCAMEL_CONSTEXPR_IF (context == explicit_context &&
  1491. BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) {
  1492. // Set flag
  1493. assert(!emptyFlags[BLOCK_SIZE - 1 -
  1494. static_cast<size_t>(i & static_cast<index_t>(BLOCK_SIZE - 1))].load(
  1495. std::memory_order_relaxed));
  1496. emptyFlags[BLOCK_SIZE - 1 - static_cast<size_t>(i & static_cast<index_t>(BLOCK_SIZE - 1))].store(
  1497. true, std::memory_order_release);
  1498. return false;
  1499. } else {
  1500. // Increment counter
  1501. auto prevVal = elementsCompletelyDequeued.fetch_add(1, std::memory_order_release);
  1502. assert(prevVal < BLOCK_SIZE);
  1503. return prevVal == BLOCK_SIZE - 1;
  1504. }
  1505. }
  1506. // Sets multiple contiguous item statuses to 'empty' (assumes no wrapping and count > 0).
  1507. // Returns true if the block is now empty (does not apply in explicit context).
  1508. template<InnerQueueContext context>
  1509. inline bool set_many_empty(MOODYCAMEL_MAYBE_UNUSED index_t i, size_t count) {
  1510. MOODYCAMEL_CONSTEXPR_IF (context == explicit_context &&
  1511. BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) {
  1512. // Set flags
  1513. std::atomic_thread_fence(std::memory_order_release);
  1514. i = BLOCK_SIZE - 1 - static_cast<size_t>(i & static_cast<index_t>(BLOCK_SIZE - 1)) - count + 1;
  1515. for (size_t j = 0; j != count; ++j) {
  1516. assert(!emptyFlags[i + j].load(std::memory_order_relaxed));
  1517. emptyFlags[i + j].store(true, std::memory_order_relaxed);
  1518. }
  1519. return false;
  1520. } else {
  1521. // Increment counter
  1522. auto prevVal = elementsCompletelyDequeued.fetch_add(count, std::memory_order_release);
  1523. assert(prevVal + count <= BLOCK_SIZE);
  1524. return prevVal + count == BLOCK_SIZE;
  1525. }
  1526. }
  1527. template<InnerQueueContext context>
  1528. inline void set_all_empty() {
  1529. MOODYCAMEL_CONSTEXPR_IF (context == explicit_context &&
  1530. BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) {
  1531. // Set all flags
  1532. for (size_t i = 0; i != BLOCK_SIZE; ++i) {
  1533. emptyFlags[i].store(true, std::memory_order_relaxed);
  1534. }
  1535. } else {
  1536. // Reset counter
  1537. elementsCompletelyDequeued.store(BLOCK_SIZE, std::memory_order_relaxed);
  1538. }
  1539. }
  1540. template<InnerQueueContext context>
  1541. inline void reset_empty() {
  1542. MOODYCAMEL_CONSTEXPR_IF (context == explicit_context &&
  1543. BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) {
  1544. // Reset flags
  1545. for (size_t i = 0; i != BLOCK_SIZE; ++i) {
  1546. emptyFlags[i].store(false, std::memory_order_relaxed);
  1547. }
  1548. } else {
  1549. // Reset counter
  1550. elementsCompletelyDequeued.store(0, std::memory_order_relaxed);
  1551. }
  1552. }
  1553. inline T *operator[](index_t idx)
  1554. MOODYCAMEL_NOEXCEPT {
  1555. return static_cast<T *>(static_cast<void *>(elements)) +
  1556. static_cast<size_t>(idx & static_cast<index_t>(BLOCK_SIZE - 1));
  1557. }
  1558. inline T const *operator[](index_t idx) const
  1559. MOODYCAMEL_NOEXCEPT {
  1560. return static_cast<T const *>(static_cast<void const *>(elements)) +
  1561. static_cast<size_t>(idx & static_cast<index_t>(BLOCK_SIZE - 1));
  1562. }
  1563. private:
  1564. static_assert(std::alignment_of<T>::value
  1565. <= sizeof(T), "The queue does not support types with an alignment greater than their size at this time");
  1566. MOODYCAMEL_ALIGNED_TYPE_LIKE(char[sizeof(T) * BLOCK_SIZE], T) elements;
  1567. public:
  1568. Block *next;
  1569. std::atomic<size_t> elementsCompletelyDequeued;
  1570. std::atomic<bool> emptyFlags[BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE : 1];
  1571. public:
  1572. std::atomic<std::uint32_t> freeListRefs;
  1573. std::atomic<Block *> freeListNext;
  1574. std::atomic<bool> shouldBeOnFreeList;
  1575. bool dynamicallyAllocated; // Perhaps a better name for this would be 'isNotPartOfInitialBlockPool'
  1576. #ifdef MCDBGQ_TRACKMEM
  1577. void* owner;
  1578. #endif
  1579. };
  1580. static_assert(std::alignment_of<Block>::value
  1581. >= std::alignment_of<T>::value, "Internal error: Blocks must be at least as aligned as the type they are wrapping");
  1582. #ifdef MCDBGQ_TRACKMEM
  1583. public:
  1584. struct MemStats;
  1585. private:
  1586. #endif
  1587. ///////////////////////////
  1588. // Producer base
  1589. ///////////////////////////
  1590. struct ProducerBase : public details::ConcurrentQueueProducerTypelessBase {
  1591. ProducerBase(ConcurrentQueue *parent_, bool isExplicit_) :
  1592. tailIndex(0),
  1593. headIndex(0),
  1594. dequeueOptimisticCount(0),
  1595. dequeueOvercommit(0),
  1596. tailBlock(nullptr),
  1597. isExplicit(isExplicit_),
  1598. parent(parent_) {
  1599. }
  1600. virtual ~ProducerBase() {}
  1601. template<typename U>
  1602. inline bool dequeue(U &element) {
  1603. if (isExplicit) {
  1604. return static_cast<ExplicitProducer *>(this)->dequeue(element);
  1605. } else {
  1606. return static_cast<ImplicitProducer *>(this)->dequeue(element);
  1607. }
  1608. }
  1609. template<typename It>
  1610. inline size_t dequeue_bulk(It &itemFirst, size_t max) {
  1611. if (isExplicit) {
  1612. return static_cast<ExplicitProducer *>(this)->dequeue_bulk(itemFirst, max);
  1613. } else {
  1614. return static_cast<ImplicitProducer *>(this)->dequeue_bulk(itemFirst, max);
  1615. }
  1616. }
  1617. inline ProducerBase *next_prod() const { return static_cast<ProducerBase *>(next); }
  1618. inline size_t size_approx() const {
  1619. auto tail = tailIndex.load(std::memory_order_relaxed);
  1620. auto head = headIndex.load(std::memory_order_relaxed);
  1621. return details::circular_less_than(head, tail) ? static_cast<size_t>(tail - head) : 0;
  1622. }
  1623. inline index_t getTail() const { return tailIndex.load(std::memory_order_relaxed); }
  1624. protected:
  1625. std::atomic<index_t> tailIndex; // Where to enqueue to next
  1626. std::atomic<index_t> headIndex; // Where to dequeue from next
  1627. std::atomic<index_t> dequeueOptimisticCount;
  1628. std::atomic<index_t> dequeueOvercommit;
  1629. Block *tailBlock;
  1630. public:
  1631. bool isExplicit;
  1632. ConcurrentQueue *parent;
  1633. protected:
  1634. #ifdef MCDBGQ_TRACKMEM
  1635. friend struct MemStats;
  1636. #endif
  1637. };
  1638. ///////////////////////////
  1639. // Explicit queue
  1640. ///////////////////////////
  1641. struct ExplicitProducer : public ProducerBase {
  1642. explicit ExplicitProducer(ConcurrentQueue *parent_) :
  1643. ProducerBase(parent_, true),
  1644. blockIndex(nullptr),
  1645. pr_blockIndexSlotsUsed(0),
  1646. pr_blockIndexSize(EXPLICIT_INITIAL_INDEX_SIZE >> 1),
  1647. pr_blockIndexFront(0),
  1648. pr_blockIndexEntries(nullptr),
  1649. pr_blockIndexRaw(nullptr) {
  1650. size_t poolBasedIndexSize = details::ceil_to_pow_2(parent_->initialBlockPoolSize) >> 1;
  1651. if (poolBasedIndexSize > pr_blockIndexSize) {
  1652. pr_blockIndexSize = poolBasedIndexSize;
  1653. }
  1654. new_block_index(
  1655. 0); // This creates an index with double the number of current entries, i.e. EXPLICIT_INITIAL_INDEX_SIZE
  1656. }
  1657. ~ExplicitProducer() {
  1658. // Destruct any elements not yet dequeued.
  1659. // Since we're in the destructor, we can assume all elements
  1660. // are either completely dequeued or completely not (no halfways).
  1661. if (this->tailBlock != nullptr) { // Note this means there must be a block index too
  1662. // First find the block that's partially dequeued, if any
  1663. Block *halfDequeuedBlock = nullptr;
  1664. if ((this->headIndex.load(std::memory_order_relaxed) & static_cast<index_t>(BLOCK_SIZE - 1)) != 0) {
  1665. // The head's not on a block boundary, meaning a block somewhere is partially dequeued
  1666. // (or the head block is the tail block and was fully dequeued, but the head/tail are still not on a boundary)
  1667. size_t i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & (pr_blockIndexSize - 1);
  1668. while (details::circular_less_than<index_t>(pr_blockIndexEntries[i].base + BLOCK_SIZE,
  1669. this->headIndex.load(std::memory_order_relaxed))) {
  1670. i = (i + 1) & (pr_blockIndexSize - 1);
  1671. }
  1672. assert(details::circular_less_than<index_t>(pr_blockIndexEntries[i].base,
  1673. this->headIndex.load(std::memory_order_relaxed)));
  1674. halfDequeuedBlock = pr_blockIndexEntries[i].block;
  1675. }
  1676. // Start at the head block (note the first line in the loop gives us the head from the tail on the first iteration)
  1677. auto block = this->tailBlock;
  1678. do {
  1679. block = block->next;
  1680. if (block->ConcurrentQueue::Block::template is_empty<explicit_context>()) {
  1681. continue;
  1682. }
  1683. size_t i = 0; // Offset into block
  1684. if (block == halfDequeuedBlock) {
  1685. i = static_cast<size_t>(this->headIndex.load(std::memory_order_relaxed) &
  1686. static_cast<index_t>(BLOCK_SIZE - 1));
  1687. }
  1688. // Walk through all the items in the block; if this is the tail block, we need to stop when we reach the tail index
  1689. auto lastValidIndex = (this->tailIndex.load(std::memory_order_relaxed) &
  1690. static_cast<index_t>(BLOCK_SIZE - 1)) == 0 ? BLOCK_SIZE
  1691. : static_cast<size_t>(
  1692. this->tailIndex.load(std::memory_order_relaxed) &
  1693. static_cast<index_t>(BLOCK_SIZE - 1));
  1694. while (i != BLOCK_SIZE && (block != this->tailBlock || i != lastValidIndex)) {
  1695. (*block)[i++]->~T();
  1696. }
  1697. } while (block != this->tailBlock);
  1698. }
  1699. // Destroy all blocks that we own
  1700. if (this->tailBlock != nullptr) {
  1701. auto block = this->tailBlock;
  1702. do {
  1703. auto nextBlock = block->next;
  1704. if (block->dynamicallyAllocated) {
  1705. destroy(block);
  1706. } else {
  1707. this->parent->add_block_to_free_list(block);
  1708. }
  1709. block = nextBlock;
  1710. } while (block != this->tailBlock);
  1711. }
  1712. // Destroy the block indices
  1713. auto header = static_cast<BlockIndexHeader *>(pr_blockIndexRaw);
  1714. while (header != nullptr) {
  1715. auto prev = static_cast<BlockIndexHeader *>(header->prev);
  1716. header->~BlockIndexHeader();
  1717. (Traits::free)(header);
  1718. header = prev;
  1719. }
  1720. }
  1721. template<AllocationMode allocMode, typename U>
  1722. inline bool enqueue(U &&element) {
  1723. index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed);
  1724. index_t newTailIndex = 1 + currentTailIndex;
  1725. if ((currentTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) == 0) {
  1726. // We reached the end of a block, start a new one
  1727. auto startBlock = this->tailBlock;
  1728. auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed;
  1729. if (this->tailBlock != nullptr &&
  1730. this->tailBlock->next->ConcurrentQueue::Block::template is_empty<explicit_context>()) {
  1731. // We can re-use the block ahead of us, it's empty!
  1732. this->tailBlock = this->tailBlock->next;
  1733. this->tailBlock->ConcurrentQueue::Block::template reset_empty<explicit_context>();
  1734. // We'll put the block on the block index (guaranteed to be room since we're conceptually removing the
  1735. // last block from it first -- except instead of removing then adding, we can just overwrite).
  1736. // Note that there must be a valid block index here, since even if allocation failed in the ctor,
  1737. // it would have been re-attempted when adding the first block to the queue; since there is such
  1738. // a block, a block index must have been successfully allocated.
  1739. } else {
  1740. // Whatever head value we see here is >= the last value we saw here (relatively),
  1741. // and <= its current value. Since we have the most recent tail, the head must be
  1742. // <= to it.
  1743. auto head = this->headIndex.load(std::memory_order_relaxed);
  1744. assert(!details::circular_less_than<index_t>(currentTailIndex, head));
  1745. if (!details::circular_less_than<index_t>(head, currentTailIndex + BLOCK_SIZE)
  1746. || (MAX_SUBQUEUE_SIZE != details::const_numeric_max<size_t>::value &&
  1747. (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) {
  1748. // We can't enqueue in another block because there's not enough leeway -- the
  1749. // tail could surpass the head by the time the block fills up! (Or we'll exceed
  1750. // the size limit, if the second part of the condition was true.)
  1751. return false;
  1752. }
  1753. // We're going to need a new block; check that the block index has room
  1754. if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize) {
  1755. // Hmm, the circular block index is already full -- we'll need
  1756. // to allocate a new index. Note pr_blockIndexRaw can only be nullptr if
  1757. // the initial allocation failed in the constructor.
  1758. MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) {
  1759. return false;
  1760. } else if (!new_block_index(pr_blockIndexSlotsUsed)) {
  1761. return false;
  1762. }
  1763. }
  1764. // Insert a new block in the circular linked list
  1765. auto newBlock = this->parent->ConcurrentQueue::template requisition_block<allocMode>();
  1766. if (newBlock == nullptr) {
  1767. return false;
  1768. }
  1769. #ifdef MCDBGQ_TRACKMEM
  1770. newBlock->owner = this;
  1771. #endif
  1772. newBlock->ConcurrentQueue::Block::template reset_empty<explicit_context>();
  1773. if (this->tailBlock == nullptr) {
  1774. newBlock->next = newBlock;
  1775. } else {
  1776. newBlock->next = this->tailBlock->next;
  1777. this->tailBlock->next = newBlock;
  1778. }
  1779. this->tailBlock = newBlock;
  1780. ++pr_blockIndexSlotsUsed;
  1781. }
  1782. MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new(static_cast<T *>(nullptr)) T(
  1783. std::forward<U>(element)))) {
  1784. // The constructor may throw. We want the element not to appear in the queue in
  1785. // that case (without corrupting the queue):
  1786. MOODYCAMEL_TRY {
  1787. new((*this->tailBlock)[currentTailIndex]) T(std::forward<U>(element));
  1788. }
  1789. MOODYCAMEL_CATCH (...) {
  1790. // Revert change to the current block, but leave the new block available
  1791. // for next time
  1792. pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed;
  1793. this->tailBlock = startBlock == nullptr ? this->tailBlock : startBlock;
  1794. MOODYCAMEL_RETHROW;
  1795. }
  1796. } else {
  1797. (void) startBlock;
  1798. (void) originalBlockIndexSlotsUsed;
  1799. }
  1800. // Add block to block index
  1801. auto &entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront];
  1802. entry.base = currentTailIndex;
  1803. entry.block = this->tailBlock;
  1804. blockIndex.load(std::memory_order_relaxed)->front.store(pr_blockIndexFront,
  1805. std::memory_order_release);
  1806. pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1);
  1807. MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new(static_cast<T *>(nullptr)) T(
  1808. std::forward<U>(element)))) {
  1809. this->tailIndex.store(newTailIndex, std::memory_order_release);
  1810. return true;
  1811. }
  1812. }
  1813. // Enqueue
  1814. new((*this->tailBlock)[currentTailIndex]) T(std::forward<U>(element));
  1815. this->tailIndex.store(newTailIndex, std::memory_order_release);
  1816. return true;
  1817. }
  1818. template<typename U>
  1819. bool dequeue(U &element) {
  1820. auto tail = this->tailIndex.load(std::memory_order_relaxed);
  1821. auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed);
  1822. if (details::circular_less_than<index_t>(
  1823. this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) {
  1824. // Might be something to dequeue, let's give it a try
  1825. // Note that this if is purely for performance purposes in the common case when the queue is
  1826. // empty and the values are eventually consistent -- we may enter here spuriously.
  1827. // Note that whatever the values of overcommit and tail are, they are not going to change (unless we
  1828. // change them) and must be the same value at this point (inside the if) as when the if condition was
  1829. // evaluated.
  1830. // We insert an acquire fence here to synchronize-with the release upon incrementing dequeueOvercommit below.
  1831. // This ensures that whatever the value we got loaded into overcommit, the load of dequeueOptisticCount in
  1832. // the fetch_add below will result in a value at least as recent as that (and therefore at least as large).
  1833. // Note that I believe a compiler (signal) fence here would be sufficient due to the nature of fetch_add (all
  1834. // read-modify-write operations are guaranteed to work on the latest value in the modification order), but
  1835. // unfortunately that can't be shown to be correct using only the C++11 standard.
  1836. // See http://stackoverflow.com/questions/18223161/what-are-the-c11-memory-ordering-guarantees-in-this-corner-case
  1837. std::atomic_thread_fence(std::memory_order_acquire);
  1838. // Increment optimistic counter, then check if it went over the boundary
  1839. auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed);
  1840. // Note that since dequeueOvercommit must be <= dequeueOptimisticCount (because dequeueOvercommit is only ever
  1841. // incremented after dequeueOptimisticCount -- this is enforced in the `else` block below), and since we now
  1842. // have a version of dequeueOptimisticCount that is at least as recent as overcommit (due to the release upon
  1843. // incrementing dequeueOvercommit and the acquire above that synchronizes with it), overcommit <= myDequeueCount.
  1844. // However, we can't assert this since both dequeueOptimisticCount and dequeueOvercommit may (independently)
  1845. // overflow; in such a case, though, the logic still holds since the difference between the two is maintained.
  1846. // Note that we reload tail here in case it changed; it will be the same value as before or greater, since
  1847. // this load is sequenced after (happens after) the earlier load above. This is supported by read-read
  1848. // coherency (as defined in the standard), explained here: http://en.cppreference.com/w/cpp/atomic/memory_order
  1849. tail = this->tailIndex.load(std::memory_order_acquire);
  1850. if ((details::likely)(details::circular_less_than<index_t>(myDequeueCount - overcommit, tail))) {
  1851. // Guaranteed to be at least one element to dequeue!
  1852. // Get the index. Note that since there's guaranteed to be at least one element, this
  1853. // will never exceed tail. We need to do an acquire-release fence here since it's possible
  1854. // that whatever condition got us to this point was for an earlier enqueued element (that
  1855. // we already see the memory effects for), but that by the time we increment somebody else
  1856. // has incremented it, and we need to see the memory effects for *that* element, which is
  1857. // in such a case is necessarily visible on the thread that incremented it in the first
  1858. // place with the more current condition (they must have acquired a tail that is at least
  1859. // as recent).
  1860. auto index = this->headIndex.fetch_add(1, std::memory_order_acq_rel);
  1861. // Determine which block the element is in
  1862. auto localBlockIndex = blockIndex.load(std::memory_order_acquire);
  1863. auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire);
  1864. // We need to be careful here about subtracting and dividing because of index wrap-around.
  1865. // When an index wraps, we need to preserve the sign of the offset when dividing it by the
  1866. // block size (in order to get a correct signed block count offset in all cases):
  1867. auto headBase = localBlockIndex->entries[localBlockIndexHead].base;
  1868. auto blockBaseIndex = index & ~static_cast<index_t>(BLOCK_SIZE - 1);
  1869. auto offset = static_cast<size_t>(
  1870. static_cast<typename std::make_signed<index_t>::type>(blockBaseIndex - headBase) /
  1871. BLOCK_SIZE);
  1872. auto block = localBlockIndex->entries[(localBlockIndexHead + offset) &
  1873. (localBlockIndex->size - 1)].block;
  1874. // Dequeue
  1875. auto &el = *((*block)[index]);
  1876. if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T &&, element = std::move(el))) {
  1877. // Make sure the element is still fully dequeued and destroyed even if the assignment
  1878. // throws
  1879. struct Guard {
  1880. Block *block;
  1881. index_t index;
  1882. ~Guard() {
  1883. (*block)[index]->~T();
  1884. block->ConcurrentQueue::Block::template set_empty<explicit_context>(index);
  1885. }
  1886. } guard = {block, index};
  1887. element = std::move(el); // NOLINT
  1888. } else {
  1889. element = std::move(el); // NOLINT
  1890. el.~T(); // NOLINT
  1891. block->ConcurrentQueue::Block::template set_empty<explicit_context>(index);
  1892. }
  1893. return true;
  1894. } else {
  1895. // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent
  1896. this->dequeueOvercommit.fetch_add(1,
  1897. std::memory_order_release); // Release so that the fetch_add on dequeueOptimisticCount is guaranteed to happen before this write
  1898. }
  1899. }
  1900. return false;
  1901. }
  1902. template<AllocationMode allocMode, typename It>
  1903. bool MOODYCAMEL_NO_TSAN enqueue_bulk(It itemFirst, size_t count) {
  1904. // First, we need to make sure we have enough room to enqueue all of the elements;
  1905. // this means pre-allocating blocks and putting them in the block index (but only if
  1906. // all the allocations succeeded).
  1907. index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed);
  1908. auto startBlock = this->tailBlock;
  1909. auto originalBlockIndexFront = pr_blockIndexFront;
  1910. auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed;
  1911. Block *firstAllocatedBlock = nullptr;
  1912. // Figure out how many blocks we'll need to allocate, and do so
  1913. size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1)) -
  1914. ((startTailIndex - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1));
  1915. index_t currentTailIndex = (startTailIndex - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1);
  1916. if (blockBaseDiff > 0) {
  1917. // Allocate as many blocks as possible from ahead
  1918. while (blockBaseDiff > 0 && this->tailBlock != nullptr &&
  1919. this->tailBlock->next != firstAllocatedBlock &&
  1920. this->tailBlock->next->ConcurrentQueue::Block::template is_empty<explicit_context>()) {
  1921. blockBaseDiff -= static_cast<index_t>(BLOCK_SIZE);
  1922. currentTailIndex += static_cast<index_t>(BLOCK_SIZE);
  1923. this->tailBlock = this->tailBlock->next;
  1924. firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock;
  1925. auto &entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront];
  1926. entry.base = currentTailIndex;
  1927. entry.block = this->tailBlock;
  1928. pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1);
  1929. }
  1930. // Now allocate as many blocks as necessary from the block pool
  1931. while (blockBaseDiff > 0) {
  1932. blockBaseDiff -= static_cast<index_t>(BLOCK_SIZE);
  1933. currentTailIndex += static_cast<index_t>(BLOCK_SIZE);
  1934. auto head = this->headIndex.load(std::memory_order_relaxed);
  1935. assert(!details::circular_less_than<index_t>(currentTailIndex, head));
  1936. bool full = !details::circular_less_than<index_t>(head, currentTailIndex + BLOCK_SIZE) ||
  1937. (MAX_SUBQUEUE_SIZE != details::const_numeric_max<size_t>::value &&
  1938. (MAX_SUBQUEUE_SIZE == 0 ||
  1939. MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head));
  1940. if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize || full) {
  1941. MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) {
  1942. // Failed to allocate, undo changes (but keep injected blocks)
  1943. pr_blockIndexFront = originalBlockIndexFront;
  1944. pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed;
  1945. this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock;
  1946. return false;
  1947. } else if (full || !new_block_index(originalBlockIndexSlotsUsed)) {
  1948. // Failed to allocate, undo changes (but keep injected blocks)
  1949. pr_blockIndexFront = originalBlockIndexFront;
  1950. pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed;
  1951. this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock;
  1952. return false;
  1953. }
  1954. // pr_blockIndexFront is updated inside new_block_index, so we need to
  1955. // update our fallback value too (since we keep the new index even if we
  1956. // later fail)
  1957. originalBlockIndexFront = originalBlockIndexSlotsUsed;
  1958. }
  1959. // Insert a new block in the circular linked list
  1960. auto newBlock = this->parent->ConcurrentQueue::template requisition_block<allocMode>();
  1961. if (newBlock == nullptr) {
  1962. pr_blockIndexFront = originalBlockIndexFront;
  1963. pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed;
  1964. this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock;
  1965. return false;
  1966. }
  1967. #ifdef MCDBGQ_TRACKMEM
  1968. newBlock->owner = this;
  1969. #endif
  1970. newBlock->ConcurrentQueue::Block::template set_all_empty<explicit_context>();
  1971. if (this->tailBlock == nullptr) {
  1972. newBlock->next = newBlock;
  1973. } else {
  1974. newBlock->next = this->tailBlock->next;
  1975. this->tailBlock->next = newBlock;
  1976. }
  1977. this->tailBlock = newBlock;
  1978. firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock;
  1979. ++pr_blockIndexSlotsUsed;
  1980. auto &entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront];
  1981. entry.base = currentTailIndex;
  1982. entry.block = this->tailBlock;
  1983. pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1);
  1984. }
  1985. // Excellent, all allocations succeeded. Reset each block's emptiness before we fill them up, and
  1986. // publish the new block index front
  1987. auto block = firstAllocatedBlock;
  1988. while (true) {
  1989. block->ConcurrentQueue::Block::template reset_empty<explicit_context>();
  1990. if (block == this->tailBlock) {
  1991. break;
  1992. }
  1993. block = block->next;
  1994. }
  1995. MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst),
  1996. new(static_cast<T *>(nullptr)) T(
  1997. details::deref_noexcept(itemFirst)))) {
  1998. blockIndex.load(std::memory_order_relaxed)->front.store(
  1999. (pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release);
  2000. }
  2001. }
  2002. // Enqueue, one block at a time
  2003. index_t newTailIndex = startTailIndex + static_cast<index_t>(count);
  2004. currentTailIndex = startTailIndex;
  2005. auto endBlock = this->tailBlock;
  2006. this->tailBlock = startBlock;
  2007. assert((startTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr ||
  2008. count == 0);
  2009. if ((startTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) {
  2010. this->tailBlock = firstAllocatedBlock;
  2011. }
  2012. while (true) {
  2013. index_t stopIndex = (currentTailIndex & ~static_cast<index_t>(BLOCK_SIZE - 1)) +
  2014. static_cast<index_t>(BLOCK_SIZE);
  2015. if (details::circular_less_than<index_t>(newTailIndex, stopIndex)) {
  2016. stopIndex = newTailIndex;
  2017. }
  2018. MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst),
  2019. new(static_cast<T *>(nullptr)) T(
  2020. details::deref_noexcept(itemFirst)))) {
  2021. while (currentTailIndex != stopIndex) {
  2022. new((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++);
  2023. }
  2024. } else {
  2025. MOODYCAMEL_TRY {
  2026. while (currentTailIndex != stopIndex) {
  2027. // Must use copy constructor even if move constructor is available
  2028. // because we may have to revert if there's an exception.
  2029. // Sorry about the horrible templated next line, but it was the only way
  2030. // to disable moving *at compile time*, which is important because a type
  2031. // may only define a (noexcept) move constructor, and so calls to the
  2032. // cctor will not compile, even if they are in an if branch that will never
  2033. // be executed
  2034. new((*this->tailBlock)[currentTailIndex]) T(
  2035. details::nomove_if<!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst),
  2036. new(static_cast<T *>(nullptr)) T(
  2037. details::deref_noexcept(
  2038. itemFirst)))>::eval(
  2039. *itemFirst));
  2040. ++currentTailIndex;
  2041. ++itemFirst;
  2042. }
  2043. }
  2044. MOODYCAMEL_CATCH (...) {
  2045. // Oh dear, an exception's been thrown -- destroy the elements that
  2046. // were enqueued so far and revert the entire bulk operation (we'll keep
  2047. // any allocated blocks in our linked list for later, though).
  2048. auto constructedStopIndex = currentTailIndex;
  2049. auto lastBlockEnqueued = this->tailBlock;
  2050. pr_blockIndexFront = originalBlockIndexFront;
  2051. pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed;
  2052. this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock;
  2053. if (!details::is_trivially_destructible<T>::value) {
  2054. auto block = startBlock;
  2055. if ((startTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) == 0) {
  2056. block = firstAllocatedBlock;
  2057. }
  2058. currentTailIndex = startTailIndex;
  2059. while (true) {
  2060. stopIndex = (currentTailIndex & ~static_cast<index_t>(BLOCK_SIZE - 1)) +
  2061. static_cast<index_t>(BLOCK_SIZE);
  2062. if (details::circular_less_than<index_t>(constructedStopIndex, stopIndex)) {
  2063. stopIndex = constructedStopIndex;
  2064. }
  2065. while (currentTailIndex != stopIndex) {
  2066. (*block)[currentTailIndex++]->~T();
  2067. }
  2068. if (block == lastBlockEnqueued) {
  2069. break;
  2070. }
  2071. block = block->next;
  2072. }
  2073. }
  2074. MOODYCAMEL_RETHROW;
  2075. }
  2076. }
  2077. if (this->tailBlock == endBlock) {
  2078. assert(currentTailIndex == newTailIndex);
  2079. break;
  2080. }
  2081. this->tailBlock = this->tailBlock->next;
  2082. }
  2083. MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst),
  2084. new(static_cast<T *>(nullptr)) T(
  2085. details::deref_noexcept(itemFirst)))) {
  2086. if (firstAllocatedBlock != nullptr)
  2087. blockIndex.load(std::memory_order_relaxed)->front.store(
  2088. (pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release);
  2089. }
  2090. this->tailIndex.store(newTailIndex, std::memory_order_release);
  2091. return true;
  2092. }
  2093. template<typename It>
  2094. size_t dequeue_bulk(It &itemFirst, size_t max) {
  2095. auto tail = this->tailIndex.load(std::memory_order_relaxed);
  2096. auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed);
  2097. auto desiredCount = static_cast<size_t>(tail -
  2098. (this->dequeueOptimisticCount.load(std::memory_order_relaxed) -
  2099. overcommit));
  2100. if (details::circular_less_than<size_t>(0, desiredCount)) {
  2101. desiredCount = desiredCount < max ? desiredCount : max;
  2102. std::atomic_thread_fence(std::memory_order_acquire);
  2103. auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount,
  2104. std::memory_order_relaxed);
  2105. tail = this->tailIndex.load(std::memory_order_acquire);
  2106. auto actualCount = static_cast<size_t>(tail - (myDequeueCount - overcommit));
  2107. if (details::circular_less_than<size_t>(0, actualCount)) {
  2108. actualCount = desiredCount < actualCount ? desiredCount : actualCount;
  2109. if (actualCount < desiredCount) {
  2110. this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release);
  2111. }
  2112. // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this
  2113. // will never exceed tail.
  2114. auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel);
  2115. // Determine which block the first element is in
  2116. auto localBlockIndex = blockIndex.load(std::memory_order_acquire);
  2117. auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire);
  2118. auto headBase = localBlockIndex->entries[localBlockIndexHead].base;
  2119. auto firstBlockBaseIndex = firstIndex & ~static_cast<index_t>(BLOCK_SIZE - 1);
  2120. auto offset = static_cast<size_t>(
  2121. static_cast<typename std::make_signed<index_t>::type>(firstBlockBaseIndex - headBase) /
  2122. BLOCK_SIZE);
  2123. auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1);
  2124. // Iterate the blocks and dequeue
  2125. auto index = firstIndex;
  2126. do {
  2127. auto firstIndexInBlock = index;
  2128. index_t endIndex =
  2129. (index & ~static_cast<index_t>(BLOCK_SIZE - 1)) + static_cast<index_t>(BLOCK_SIZE);
  2130. endIndex = details::circular_less_than<index_t>(
  2131. firstIndex + static_cast<index_t>(actualCount), endIndex) ? firstIndex +
  2132. static_cast<index_t>(actualCount)
  2133. : endIndex;
  2134. auto block = localBlockIndex->entries[indexIndex].block;
  2135. if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T &&, details::deref_noexcept(itemFirst) = std::move(
  2136. (*(*block)[index])))) {
  2137. while (index != endIndex) {
  2138. auto &el = *((*block)[index]);
  2139. *itemFirst++ = std::move(el);
  2140. el.~T();
  2141. ++index;
  2142. }
  2143. } else {
  2144. MOODYCAMEL_TRY {
  2145. while (index != endIndex) {
  2146. auto &el = *((*block)[index]);
  2147. *itemFirst = std::move(el);
  2148. ++itemFirst;
  2149. el.~T();
  2150. ++index;
  2151. }
  2152. }
  2153. MOODYCAMEL_CATCH (...) {
  2154. // It's too late to revert the dequeue, but we can make sure that all
  2155. // the dequeued objects are properly destroyed and the block index
  2156. // (and empty count) are properly updated before we propagate the exception
  2157. do {
  2158. block = localBlockIndex->entries[indexIndex].block;
  2159. while (index != endIndex) {
  2160. (*block)[index++]->~T();
  2161. }
  2162. block->ConcurrentQueue::Block::template set_many_empty<explicit_context>(
  2163. firstIndexInBlock, static_cast<size_t>(endIndex - firstIndexInBlock));
  2164. indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1);
  2165. firstIndexInBlock = index;
  2166. endIndex = (index & ~static_cast<index_t>(BLOCK_SIZE - 1)) +
  2167. static_cast<index_t>(BLOCK_SIZE);
  2168. endIndex = details::circular_less_than<index_t>(
  2169. firstIndex + static_cast<index_t>(actualCount), endIndex) ? firstIndex +
  2170. static_cast<index_t>(actualCount)
  2171. : endIndex;
  2172. } while (index != firstIndex + actualCount);
  2173. MOODYCAMEL_RETHROW;
  2174. }
  2175. }
  2176. block->ConcurrentQueue::Block::template set_many_empty<explicit_context>(firstIndexInBlock,
  2177. static_cast<size_t>(
  2178. endIndex -
  2179. firstIndexInBlock));
  2180. indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1);
  2181. } while (index != firstIndex + actualCount);
  2182. return actualCount;
  2183. } else {
  2184. // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent
  2185. this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release);
  2186. }
  2187. }
  2188. return 0;
  2189. }
  2190. private:
  2191. struct BlockIndexEntry {
  2192. index_t base;
  2193. Block *block;
  2194. };
  2195. struct BlockIndexHeader {
  2196. size_t size;
  2197. std::atomic<size_t> front; // Current slot (not next, like pr_blockIndexFront)
  2198. BlockIndexEntry *entries;
  2199. void *prev;
  2200. };
  2201. bool new_block_index(size_t numberOfFilledSlotsToExpose) {
  2202. auto prevBlockSizeMask = pr_blockIndexSize - 1;
  2203. // Create the new block
  2204. pr_blockIndexSize <<= 1;
  2205. auto newRawPtr = static_cast<char *>((Traits::malloc)(
  2206. sizeof(BlockIndexHeader) + std::alignment_of<BlockIndexEntry>::value - 1 +
  2207. sizeof(BlockIndexEntry) * pr_blockIndexSize));
  2208. if (newRawPtr == nullptr) {
  2209. pr_blockIndexSize >>= 1; // Reset to allow graceful retry
  2210. return false;
  2211. }
  2212. auto newBlockIndexEntries = reinterpret_cast<BlockIndexEntry *>(details::align_for<BlockIndexEntry>(
  2213. newRawPtr + sizeof(BlockIndexHeader)));
  2214. // Copy in all the old indices, if any
  2215. size_t j = 0;
  2216. if (pr_blockIndexSlotsUsed != 0) {
  2217. auto i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & prevBlockSizeMask;
  2218. do {
  2219. newBlockIndexEntries[j++] = pr_blockIndexEntries[i];
  2220. i = (i + 1) & prevBlockSizeMask;
  2221. } while (i != pr_blockIndexFront);
  2222. }
  2223. // Update everything
  2224. auto header = new(newRawPtr) BlockIndexHeader;
  2225. header->size = pr_blockIndexSize;
  2226. header->front.store(numberOfFilledSlotsToExpose - 1, std::memory_order_relaxed);
  2227. header->entries = newBlockIndexEntries;
  2228. header->prev = pr_blockIndexRaw; // we link the new block to the old one so we can free it later
  2229. pr_blockIndexFront = j;
  2230. pr_blockIndexEntries = newBlockIndexEntries;
  2231. pr_blockIndexRaw = newRawPtr;
  2232. blockIndex.store(header, std::memory_order_release);
  2233. return true;
  2234. }
  2235. private:
  2236. std::atomic<BlockIndexHeader *> blockIndex;
  2237. // To be used by producer only -- consumer must use the ones in referenced by blockIndex
  2238. size_t pr_blockIndexSlotsUsed;
  2239. size_t pr_blockIndexSize;
  2240. size_t pr_blockIndexFront; // Next slot (not current)
  2241. BlockIndexEntry *pr_blockIndexEntries;
  2242. void *pr_blockIndexRaw;
  2243. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  2244. public:
  2245. ExplicitProducer* nextExplicitProducer;
  2246. private:
  2247. #endif
  2248. #ifdef MCDBGQ_TRACKMEM
  2249. friend struct MemStats;
  2250. #endif
  2251. };
  2252. //////////////////////////////////
  2253. // Implicit queue
  2254. //////////////////////////////////
  2255. struct ImplicitProducer : public ProducerBase {
  2256. ImplicitProducer(ConcurrentQueue *parent_) :
  2257. ProducerBase(parent_, false),
  2258. nextBlockIndexCapacity(IMPLICIT_INITIAL_INDEX_SIZE),
  2259. blockIndex(nullptr) {
  2260. new_block_index();
  2261. }
  2262. ~ImplicitProducer() {
  2263. // Note that since we're in the destructor we can assume that all enqueue/dequeue operations
  2264. // completed already; this means that all undequeued elements are placed contiguously across
  2265. // contiguous blocks, and that only the first and last remaining blocks can be only partially
  2266. // empty (all other remaining blocks must be completely full).
  2267. #ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  2268. // Unregister ourselves for thread termination notification
  2269. if (!this->inactive.load(std::memory_order_relaxed)) {
  2270. details::ThreadExitNotifier::unsubscribe(&threadExitListener);
  2271. }
  2272. #endif
  2273. // Destroy all remaining elements!
  2274. auto tail = this->tailIndex.load(std::memory_order_relaxed);
  2275. auto index = this->headIndex.load(std::memory_order_relaxed);
  2276. Block *block = nullptr;
  2277. assert(index == tail || details::circular_less_than(index, tail));
  2278. bool forceFreeLastBlock =
  2279. index != tail; // If we enter the loop, then the last (tail) block will not be freed
  2280. while (index != tail) {
  2281. if ((index & static_cast<index_t>(BLOCK_SIZE - 1)) == 0 || block == nullptr) {
  2282. if (block != nullptr) {
  2283. // Free the old block
  2284. this->parent->add_block_to_free_list(block);
  2285. }
  2286. block = get_block_index_entry_for_index(index)->value.load(std::memory_order_relaxed);
  2287. }
  2288. ((*block)[index])->~T();
  2289. ++index;
  2290. }
  2291. // Even if the queue is empty, there's still one block that's not on the free list
  2292. // (unless the head index reached the end of it, in which case the tail will be poised
  2293. // to create a new block).
  2294. if (this->tailBlock != nullptr &&
  2295. (forceFreeLastBlock || (tail & static_cast<index_t>(BLOCK_SIZE - 1)) != 0)) {
  2296. this->parent->add_block_to_free_list(this->tailBlock);
  2297. }
  2298. // Destroy block index
  2299. auto localBlockIndex = blockIndex.load(std::memory_order_relaxed);
  2300. if (localBlockIndex != nullptr) {
  2301. for (size_t i = 0; i != localBlockIndex->capacity; ++i) {
  2302. localBlockIndex->index[i]->~BlockIndexEntry();
  2303. }
  2304. do {
  2305. auto prev = localBlockIndex->prev;
  2306. localBlockIndex->~BlockIndexHeader();
  2307. (Traits::free)(localBlockIndex);
  2308. localBlockIndex = prev;
  2309. } while (localBlockIndex != nullptr);
  2310. }
  2311. }
  2312. template<AllocationMode allocMode, typename U>
  2313. inline bool enqueue(U &&element) {
  2314. index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed);
  2315. index_t newTailIndex = 1 + currentTailIndex;
  2316. if ((currentTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) == 0) {
  2317. // We reached the end of a block, start a new one
  2318. auto head = this->headIndex.load(std::memory_order_relaxed);
  2319. assert(!details::circular_less_than<index_t>(currentTailIndex, head));
  2320. if (!details::circular_less_than<index_t>(head, currentTailIndex + BLOCK_SIZE) ||
  2321. (MAX_SUBQUEUE_SIZE != details::const_numeric_max<size_t>::value &&
  2322. (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) {
  2323. return false;
  2324. }
  2325. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2326. debug::DebugLock lock(mutex);
  2327. #endif
  2328. // Find out where we'll be inserting this block in the block index
  2329. BlockIndexEntry *idxEntry;
  2330. if (!insert_block_index_entry<allocMode>(idxEntry, currentTailIndex)) {
  2331. return false;
  2332. }
  2333. // Get ahold of a new block
  2334. auto newBlock = this->parent->ConcurrentQueue::template requisition_block<allocMode>();
  2335. if (newBlock == nullptr) {
  2336. rewind_block_index_tail();
  2337. idxEntry->value.store(nullptr, std::memory_order_relaxed);
  2338. return false;
  2339. }
  2340. #ifdef MCDBGQ_TRACKMEM
  2341. newBlock->owner = this;
  2342. #endif
  2343. newBlock->ConcurrentQueue::Block::template reset_empty<implicit_context>();
  2344. MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new(static_cast<T *>(nullptr)) T(
  2345. std::forward<U>(element)))) {
  2346. // May throw, try to insert now before we publish the fact that we have this new block
  2347. MOODYCAMEL_TRY {
  2348. new((*newBlock)[currentTailIndex]) T(std::forward<U>(element));
  2349. }
  2350. MOODYCAMEL_CATCH (...) {
  2351. rewind_block_index_tail();
  2352. idxEntry->value.store(nullptr, std::memory_order_relaxed);
  2353. this->parent->add_block_to_free_list(newBlock);
  2354. MOODYCAMEL_RETHROW;
  2355. }
  2356. }
  2357. // Insert the new block into the index
  2358. idxEntry->value.store(newBlock, std::memory_order_relaxed);
  2359. this->tailBlock = newBlock;
  2360. MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new(static_cast<T *>(nullptr)) T(
  2361. std::forward<U>(element)))) {
  2362. this->tailIndex.store(newTailIndex, std::memory_order_release);
  2363. return true;
  2364. }
  2365. }
  2366. // Enqueue
  2367. new((*this->tailBlock)[currentTailIndex]) T(std::forward<U>(element));
  2368. this->tailIndex.store(newTailIndex, std::memory_order_release);
  2369. return true;
  2370. }
  2371. template<typename U>
  2372. bool dequeue(U &element) {
  2373. // See ExplicitProducer::dequeue for rationale and explanation
  2374. index_t tail = this->tailIndex.load(std::memory_order_relaxed);
  2375. index_t overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed);
  2376. if (details::circular_less_than<index_t>(
  2377. this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) {
  2378. std::atomic_thread_fence(std::memory_order_acquire);
  2379. index_t myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed);
  2380. tail = this->tailIndex.load(std::memory_order_acquire);
  2381. if ((details::likely)(details::circular_less_than<index_t>(myDequeueCount - overcommit, tail))) {
  2382. index_t index = this->headIndex.fetch_add(1, std::memory_order_acq_rel);
  2383. // Determine which block the element is in
  2384. auto entry = get_block_index_entry_for_index(index);
  2385. // Dequeue
  2386. auto block = entry->value.load(std::memory_order_relaxed);
  2387. auto &el = *((*block)[index]);
  2388. if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T &&, element = std::move(el))) {
  2389. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2390. // Note: Acquiring the mutex with every dequeue instead of only when a block
  2391. // is released is very sub-optimal, but it is, after all, purely debug code.
  2392. debug::DebugLock lock(producer->mutex);
  2393. #endif
  2394. struct Guard {
  2395. Block *block;
  2396. index_t index;
  2397. BlockIndexEntry *entry;
  2398. ConcurrentQueue *parent;
  2399. ~Guard() {
  2400. (*block)[index]->~T();
  2401. if (block->ConcurrentQueue::Block::template set_empty<implicit_context>(index)) {
  2402. entry->value.store(nullptr, std::memory_order_relaxed);
  2403. parent->add_block_to_free_list(block);
  2404. }
  2405. }
  2406. } guard = {block, index, entry, this->parent};
  2407. element = std::move(el); // NOLINT
  2408. } else {
  2409. element = std::move(el); // NOLINT
  2410. el.~T(); // NOLINT
  2411. if (block->ConcurrentQueue::Block::template set_empty<implicit_context>(index)) {
  2412. {
  2413. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2414. debug::DebugLock lock(mutex);
  2415. #endif
  2416. // Add the block back into the global free pool (and remove from block index)
  2417. entry->value.store(nullptr, std::memory_order_relaxed);
  2418. }
  2419. this->parent->add_block_to_free_list(block); // releases the above store
  2420. }
  2421. }
  2422. return true;
  2423. } else {
  2424. this->dequeueOvercommit.fetch_add(1, std::memory_order_release);
  2425. }
  2426. }
  2427. return false;
  2428. }
  2429. #ifdef _MSC_VER
  2430. #pragma warning(push)
  2431. #pragma warning(disable: 4706) // assignment within conditional expression
  2432. #endif
  2433. template<AllocationMode allocMode, typename It>
  2434. bool enqueue_bulk(It itemFirst, size_t count) {
  2435. // First, we need to make sure we have enough room to enqueue all of the elements;
  2436. // this means pre-allocating blocks and putting them in the block index (but only if
  2437. // all the allocations succeeded).
  2438. // Note that the tailBlock we start off with may not be owned by us any more;
  2439. // this happens if it was filled up exactly to the top (setting tailIndex to
  2440. // the first index of the next block which is not yet allocated), then dequeued
  2441. // completely (putting it on the free list) before we enqueue again.
  2442. index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed);
  2443. auto startBlock = this->tailBlock;
  2444. Block *firstAllocatedBlock = nullptr;
  2445. auto endBlock = this->tailBlock;
  2446. // Figure out how many blocks we'll need to allocate, and do so
  2447. size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1)) -
  2448. ((startTailIndex - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1));
  2449. index_t currentTailIndex = (startTailIndex - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1);
  2450. if (blockBaseDiff > 0) {
  2451. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2452. debug::DebugLock lock(mutex);
  2453. #endif
  2454. do {
  2455. blockBaseDiff -= static_cast<index_t>(BLOCK_SIZE);
  2456. currentTailIndex += static_cast<index_t>(BLOCK_SIZE);
  2457. // Find out where we'll be inserting this block in the block index
  2458. BlockIndexEntry *idxEntry = nullptr; // initialization here unnecessary but compiler can't always tell
  2459. Block *newBlock;
  2460. bool indexInserted = false;
  2461. auto head = this->headIndex.load(std::memory_order_relaxed);
  2462. assert(!details::circular_less_than<index_t>(currentTailIndex, head));
  2463. bool full = !details::circular_less_than<index_t>(head, currentTailIndex + BLOCK_SIZE) ||
  2464. (MAX_SUBQUEUE_SIZE != details::const_numeric_max<size_t>::value &&
  2465. (MAX_SUBQUEUE_SIZE == 0 ||
  2466. MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head));
  2467. if (full ||
  2468. !(indexInserted = insert_block_index_entry<allocMode>(idxEntry, currentTailIndex)) ||
  2469. (newBlock = this->parent->ConcurrentQueue::template requisition_block<allocMode>()) ==
  2470. nullptr) {
  2471. // Index allocation or block allocation failed; revert any other allocations
  2472. // and index insertions done so far for this operation
  2473. if (indexInserted) {
  2474. rewind_block_index_tail();
  2475. idxEntry->value.store(nullptr, std::memory_order_relaxed);
  2476. }
  2477. currentTailIndex = (startTailIndex - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1);
  2478. for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) {
  2479. currentTailIndex += static_cast<index_t>(BLOCK_SIZE);
  2480. idxEntry = get_block_index_entry_for_index(currentTailIndex);
  2481. idxEntry->value.store(nullptr, std::memory_order_relaxed);
  2482. rewind_block_index_tail();
  2483. }
  2484. this->parent->add_blocks_to_free_list(firstAllocatedBlock);
  2485. this->tailBlock = startBlock;
  2486. return false;
  2487. }
  2488. #ifdef MCDBGQ_TRACKMEM
  2489. newBlock->owner = this;
  2490. #endif
  2491. newBlock->ConcurrentQueue::Block::template reset_empty<implicit_context>();
  2492. newBlock->next = nullptr;
  2493. // Insert the new block into the index
  2494. idxEntry->value.store(newBlock, std::memory_order_relaxed);
  2495. // Store the chain of blocks so that we can undo if later allocations fail,
  2496. // and so that we can find the blocks when we do the actual enqueueing
  2497. if ((startTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) != 0 ||
  2498. firstAllocatedBlock != nullptr) {
  2499. assert(this->tailBlock != nullptr);
  2500. this->tailBlock->next = newBlock;
  2501. }
  2502. this->tailBlock = newBlock;
  2503. endBlock = newBlock;
  2504. firstAllocatedBlock = firstAllocatedBlock == nullptr ? newBlock : firstAllocatedBlock;
  2505. } while (blockBaseDiff > 0);
  2506. }
  2507. // Enqueue, one block at a time
  2508. index_t newTailIndex = startTailIndex + static_cast<index_t>(count);
  2509. currentTailIndex = startTailIndex;
  2510. this->tailBlock = startBlock;
  2511. assert((startTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr ||
  2512. count == 0);
  2513. if ((startTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) {
  2514. this->tailBlock = firstAllocatedBlock;
  2515. }
  2516. while (true) {
  2517. index_t stopIndex = (currentTailIndex & ~static_cast<index_t>(BLOCK_SIZE - 1)) +
  2518. static_cast<index_t>(BLOCK_SIZE);
  2519. if (details::circular_less_than<index_t>(newTailIndex, stopIndex)) {
  2520. stopIndex = newTailIndex;
  2521. }
  2522. MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst),
  2523. new(static_cast<T *>(nullptr)) T(
  2524. details::deref_noexcept(itemFirst)))) {
  2525. while (currentTailIndex != stopIndex) {
  2526. new((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++);
  2527. }
  2528. } else {
  2529. MOODYCAMEL_TRY {
  2530. while (currentTailIndex != stopIndex) {
  2531. new((*this->tailBlock)[currentTailIndex]) T(
  2532. details::nomove_if<!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst),
  2533. new(static_cast<T *>(nullptr)) T(
  2534. details::deref_noexcept(
  2535. itemFirst)))>::eval(
  2536. *itemFirst));
  2537. ++currentTailIndex;
  2538. ++itemFirst;
  2539. }
  2540. }
  2541. MOODYCAMEL_CATCH (...) {
  2542. auto constructedStopIndex = currentTailIndex;
  2543. auto lastBlockEnqueued = this->tailBlock;
  2544. if (!details::is_trivially_destructible<T>::value) {
  2545. auto block = startBlock;
  2546. if ((startTailIndex & static_cast<index_t>(BLOCK_SIZE - 1)) == 0) {
  2547. block = firstAllocatedBlock;
  2548. }
  2549. currentTailIndex = startTailIndex;
  2550. while (true) {
  2551. stopIndex = (currentTailIndex & ~static_cast<index_t>(BLOCK_SIZE - 1)) +
  2552. static_cast<index_t>(BLOCK_SIZE);
  2553. if (details::circular_less_than<index_t>(constructedStopIndex, stopIndex)) {
  2554. stopIndex = constructedStopIndex;
  2555. }
  2556. while (currentTailIndex != stopIndex) {
  2557. (*block)[currentTailIndex++]->~T();
  2558. }
  2559. if (block == lastBlockEnqueued) {
  2560. break;
  2561. }
  2562. block = block->next;
  2563. }
  2564. }
  2565. currentTailIndex = (startTailIndex - 1) & ~static_cast<index_t>(BLOCK_SIZE - 1);
  2566. for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) {
  2567. currentTailIndex += static_cast<index_t>(BLOCK_SIZE);
  2568. auto idxEntry = get_block_index_entry_for_index(currentTailIndex);
  2569. idxEntry->value.store(nullptr, std::memory_order_relaxed);
  2570. rewind_block_index_tail();
  2571. }
  2572. this->parent->add_blocks_to_free_list(firstAllocatedBlock);
  2573. this->tailBlock = startBlock;
  2574. MOODYCAMEL_RETHROW;
  2575. }
  2576. }
  2577. if (this->tailBlock == endBlock) {
  2578. assert(currentTailIndex == newTailIndex);
  2579. break;
  2580. }
  2581. this->tailBlock = this->tailBlock->next;
  2582. }
  2583. this->tailIndex.store(newTailIndex, std::memory_order_release);
  2584. return true;
  2585. }
  2586. #ifdef _MSC_VER
  2587. #pragma warning(pop)
  2588. #endif
  2589. template<typename It>
  2590. size_t dequeue_bulk(It &itemFirst, size_t max) {
  2591. auto tail = this->tailIndex.load(std::memory_order_relaxed);
  2592. auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed);
  2593. auto desiredCount = static_cast<size_t>(tail -
  2594. (this->dequeueOptimisticCount.load(std::memory_order_relaxed) -
  2595. overcommit));
  2596. if (details::circular_less_than<size_t>(0, desiredCount)) {
  2597. desiredCount = desiredCount < max ? desiredCount : max;
  2598. std::atomic_thread_fence(std::memory_order_acquire);
  2599. auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount,
  2600. std::memory_order_relaxed);
  2601. tail = this->tailIndex.load(std::memory_order_acquire);
  2602. auto actualCount = static_cast<size_t>(tail - (myDequeueCount - overcommit));
  2603. if (details::circular_less_than<size_t>(0, actualCount)) {
  2604. actualCount = desiredCount < actualCount ? desiredCount : actualCount;
  2605. if (actualCount < desiredCount) {
  2606. this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release);
  2607. }
  2608. // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this
  2609. // will never exceed tail.
  2610. auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel);
  2611. // Iterate the blocks and dequeue
  2612. auto index = firstIndex;
  2613. BlockIndexHeader *localBlockIndex;
  2614. auto indexIndex = get_block_index_index_for_index(index, localBlockIndex);
  2615. do {
  2616. auto blockStartIndex = index;
  2617. index_t endIndex =
  2618. (index & ~static_cast<index_t>(BLOCK_SIZE - 1)) + static_cast<index_t>(BLOCK_SIZE);
  2619. endIndex = details::circular_less_than<index_t>(
  2620. firstIndex + static_cast<index_t>(actualCount), endIndex) ? firstIndex +
  2621. static_cast<index_t>(actualCount)
  2622. : endIndex;
  2623. auto entry = localBlockIndex->index[indexIndex];
  2624. auto block = entry->value.load(std::memory_order_relaxed);
  2625. if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T &&, details::deref_noexcept(itemFirst) = std::move(
  2626. (*(*block)[index])))) {
  2627. while (index != endIndex) {
  2628. auto &el = *((*block)[index]);
  2629. *itemFirst++ = std::move(el);
  2630. el.~T();
  2631. ++index;
  2632. }
  2633. } else {
  2634. MOODYCAMEL_TRY {
  2635. while (index != endIndex) {
  2636. auto &el = *((*block)[index]);
  2637. *itemFirst = std::move(el);
  2638. ++itemFirst;
  2639. el.~T();
  2640. ++index;
  2641. }
  2642. }
  2643. MOODYCAMEL_CATCH (...) {
  2644. do {
  2645. entry = localBlockIndex->index[indexIndex];
  2646. block = entry->value.load(std::memory_order_relaxed);
  2647. while (index != endIndex) {
  2648. (*block)[index++]->~T();
  2649. }
  2650. if (block->ConcurrentQueue::Block::template set_many_empty<implicit_context>(
  2651. blockStartIndex, static_cast<size_t>(endIndex - blockStartIndex))) {
  2652. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2653. debug::DebugLock lock(mutex);
  2654. #endif
  2655. entry->value.store(nullptr, std::memory_order_relaxed);
  2656. this->parent->add_block_to_free_list(block);
  2657. }
  2658. indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1);
  2659. blockStartIndex = index;
  2660. endIndex = (index & ~static_cast<index_t>(BLOCK_SIZE - 1)) +
  2661. static_cast<index_t>(BLOCK_SIZE);
  2662. endIndex = details::circular_less_than<index_t>(
  2663. firstIndex + static_cast<index_t>(actualCount), endIndex) ? firstIndex +
  2664. static_cast<index_t>(actualCount)
  2665. : endIndex;
  2666. } while (index != firstIndex + actualCount);
  2667. MOODYCAMEL_RETHROW;
  2668. }
  2669. }
  2670. if (block->ConcurrentQueue::Block::template set_many_empty<implicit_context>(
  2671. blockStartIndex, static_cast<size_t>(endIndex - blockStartIndex))) {
  2672. {
  2673. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2674. debug::DebugLock lock(mutex);
  2675. #endif
  2676. // Note that the set_many_empty above did a release, meaning that anybody who acquires the block
  2677. // we're about to free can use it safely since our writes (and reads!) will have happened-before then.
  2678. entry->value.store(nullptr, std::memory_order_relaxed);
  2679. }
  2680. this->parent->add_block_to_free_list(block); // releases the above store
  2681. }
  2682. indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1);
  2683. } while (index != firstIndex + actualCount);
  2684. return actualCount;
  2685. } else {
  2686. this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release);
  2687. }
  2688. }
  2689. return 0;
  2690. }
  2691. private:
  2692. // The block size must be > 1, so any number with the low bit set is an invalid block base index
  2693. static const index_t INVALID_BLOCK_BASE = 1;
  2694. struct BlockIndexEntry {
  2695. std::atomic<index_t> key;
  2696. std::atomic<Block *> value;
  2697. };
  2698. struct BlockIndexHeader {
  2699. size_t capacity;
  2700. std::atomic<size_t> tail;
  2701. BlockIndexEntry *entries;
  2702. BlockIndexEntry **index;
  2703. BlockIndexHeader *prev;
  2704. };
  2705. template<AllocationMode allocMode>
  2706. inline bool insert_block_index_entry(BlockIndexEntry *&idxEntry, index_t blockStartIndex) {
  2707. auto localBlockIndex = blockIndex.load(
  2708. std::memory_order_relaxed); // We're the only writer thread, relaxed is OK
  2709. if (localBlockIndex == nullptr) {
  2710. return false; // this can happen if new_block_index failed in the constructor
  2711. }
  2712. size_t newTail =
  2713. (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1);
  2714. idxEntry = localBlockIndex->index[newTail];
  2715. if (idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE ||
  2716. idxEntry->value.load(std::memory_order_relaxed) == nullptr) {
  2717. idxEntry->key.store(blockStartIndex, std::memory_order_relaxed);
  2718. localBlockIndex->tail.store(newTail, std::memory_order_release);
  2719. return true;
  2720. }
  2721. // No room in the old block index, try to allocate another one!
  2722. MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) {
  2723. return false;
  2724. } else if (!new_block_index()) {
  2725. return false;
  2726. } else {
  2727. localBlockIndex = blockIndex.load(std::memory_order_relaxed);
  2728. newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) &
  2729. (localBlockIndex->capacity - 1);
  2730. idxEntry = localBlockIndex->index[newTail];
  2731. assert(idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE);
  2732. idxEntry->key.store(blockStartIndex, std::memory_order_relaxed);
  2733. localBlockIndex->tail.store(newTail, std::memory_order_release);
  2734. return true;
  2735. }
  2736. }
  2737. inline void rewind_block_index_tail() {
  2738. auto localBlockIndex = blockIndex.load(std::memory_order_relaxed);
  2739. localBlockIndex->tail.store(
  2740. (localBlockIndex->tail.load(std::memory_order_relaxed) - 1) & (localBlockIndex->capacity - 1),
  2741. std::memory_order_relaxed);
  2742. }
  2743. inline BlockIndexEntry *get_block_index_entry_for_index(index_t index) const {
  2744. BlockIndexHeader *localBlockIndex;
  2745. auto idx = get_block_index_index_for_index(index, localBlockIndex);
  2746. return localBlockIndex->index[idx];
  2747. }
  2748. inline size_t get_block_index_index_for_index(index_t index, BlockIndexHeader *&localBlockIndex) const {
  2749. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2750. debug::DebugLock lock(mutex);
  2751. #endif
  2752. index &= ~static_cast<index_t>(BLOCK_SIZE - 1);
  2753. localBlockIndex = blockIndex.load(std::memory_order_acquire);
  2754. auto tail = localBlockIndex->tail.load(std::memory_order_acquire);
  2755. auto tailBase = localBlockIndex->index[tail]->key.load(std::memory_order_relaxed);
  2756. assert(tailBase != INVALID_BLOCK_BASE);
  2757. // Note: Must use division instead of shift because the index may wrap around, causing a negative
  2758. // offset, whose negativity we want to preserve
  2759. auto offset = static_cast<size_t>(
  2760. static_cast<typename std::make_signed<index_t>::type>(index - tailBase) / BLOCK_SIZE);
  2761. size_t idx = (tail + offset) & (localBlockIndex->capacity - 1);
  2762. assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index &&
  2763. localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr);
  2764. return idx;
  2765. }
  2766. bool new_block_index() {
  2767. auto prev = blockIndex.load(std::memory_order_relaxed);
  2768. size_t prevCapacity = prev == nullptr ? 0 : prev->capacity;
  2769. auto entryCount = prev == nullptr ? nextBlockIndexCapacity : prevCapacity;
  2770. auto raw = static_cast<char *>((Traits::malloc)(
  2771. sizeof(BlockIndexHeader) +
  2772. std::alignment_of<BlockIndexEntry>::value - 1 + sizeof(BlockIndexEntry) * entryCount +
  2773. std::alignment_of<BlockIndexEntry *>::value - 1 +
  2774. sizeof(BlockIndexEntry *) * nextBlockIndexCapacity));
  2775. if (raw == nullptr) {
  2776. return false;
  2777. }
  2778. auto header = new(raw) BlockIndexHeader;
  2779. auto entries = reinterpret_cast<BlockIndexEntry *>(details::align_for<BlockIndexEntry>(
  2780. raw + sizeof(BlockIndexHeader)));
  2781. auto index = reinterpret_cast<BlockIndexEntry **>(details::align_for<BlockIndexEntry *>(
  2782. reinterpret_cast<char *>(entries) + sizeof(BlockIndexEntry) * entryCount));
  2783. if (prev != nullptr) {
  2784. auto prevTail = prev->tail.load(std::memory_order_relaxed);
  2785. auto prevPos = prevTail;
  2786. size_t i = 0;
  2787. do {
  2788. prevPos = (prevPos + 1) & (prev->capacity - 1);
  2789. index[i++] = prev->index[prevPos];
  2790. } while (prevPos != prevTail);
  2791. assert(i == prevCapacity);
  2792. }
  2793. for (size_t i = 0; i != entryCount; ++i) {
  2794. new(entries + i) BlockIndexEntry;
  2795. entries[i].key.store(INVALID_BLOCK_BASE, std::memory_order_relaxed);
  2796. index[prevCapacity + i] = entries + i;
  2797. }
  2798. header->prev = prev;
  2799. header->entries = entries;
  2800. header->index = index;
  2801. header->capacity = nextBlockIndexCapacity;
  2802. header->tail.store((prevCapacity - 1) & (nextBlockIndexCapacity - 1), std::memory_order_relaxed);
  2803. blockIndex.store(header, std::memory_order_release);
  2804. nextBlockIndexCapacity <<= 1;
  2805. return true;
  2806. }
  2807. private:
  2808. size_t nextBlockIndexCapacity;
  2809. std::atomic<BlockIndexHeader *> blockIndex;
  2810. #ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  2811. public:
  2812. details::ThreadExitListener threadExitListener;
  2813. private:
  2814. #endif
  2815. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  2816. public:
  2817. ImplicitProducer* nextImplicitProducer;
  2818. private:
  2819. #endif
  2820. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
  2821. mutable debug::DebugMutex mutex;
  2822. #endif
  2823. #ifdef MCDBGQ_TRACKMEM
  2824. friend struct MemStats;
  2825. #endif
  2826. };
  2827. //////////////////////////////////
  2828. // Block pool manipulation
  2829. //////////////////////////////////
  2830. void populate_initial_block_list(size_t blockCount) {
  2831. initialBlockPoolSize = blockCount;
  2832. if (initialBlockPoolSize == 0) {
  2833. initialBlockPool = nullptr;
  2834. return;
  2835. }
  2836. initialBlockPool = create_array<Block>(blockCount);
  2837. if (initialBlockPool == nullptr) {
  2838. initialBlockPoolSize = 0;
  2839. }
  2840. for (size_t i = 0; i < initialBlockPoolSize; ++i) {
  2841. initialBlockPool[i].dynamicallyAllocated = false;
  2842. }
  2843. }
  2844. inline Block *try_get_block_from_initial_pool() {
  2845. if (initialBlockPoolIndex.load(std::memory_order_relaxed) >= initialBlockPoolSize) {
  2846. return nullptr;
  2847. }
  2848. auto index = initialBlockPoolIndex.fetch_add(1, std::memory_order_relaxed);
  2849. return index < initialBlockPoolSize ? (initialBlockPool + index) : nullptr;
  2850. }
  2851. inline void add_block_to_free_list(Block *block) {
  2852. #ifdef MCDBGQ_TRACKMEM
  2853. block->owner = nullptr;
  2854. #endif
  2855. freeList.add(block);
  2856. }
  2857. inline void add_blocks_to_free_list(Block *block) {
  2858. while (block != nullptr) {
  2859. auto next = block->next;
  2860. add_block_to_free_list(block);
  2861. block = next;
  2862. }
  2863. }
  2864. inline Block *try_get_block_from_free_list() {
  2865. return freeList.try_get();
  2866. }
  2867. // Gets a free block from one of the memory pools, or allocates a new one (if applicable)
  2868. template<AllocationMode canAlloc>
  2869. Block *requisition_block() {
  2870. auto block = try_get_block_from_initial_pool();
  2871. if (block != nullptr) {
  2872. return block;
  2873. }
  2874. block = try_get_block_from_free_list();
  2875. if (block != nullptr) {
  2876. return block;
  2877. }
  2878. MOODYCAMEL_CONSTEXPR_IF (canAlloc == CanAlloc) {
  2879. return create<Block>();
  2880. } else {
  2881. return nullptr;
  2882. }
  2883. }
  2884. #ifdef MCDBGQ_TRACKMEM
  2885. public:
  2886. struct MemStats {
  2887. size_t allocatedBlocks;
  2888. size_t usedBlocks;
  2889. size_t freeBlocks;
  2890. size_t ownedBlocksExplicit;
  2891. size_t ownedBlocksImplicit;
  2892. size_t implicitProducers;
  2893. size_t explicitProducers;
  2894. size_t elementsEnqueued;
  2895. size_t blockClassBytes;
  2896. size_t queueClassBytes;
  2897. size_t implicitBlockIndexBytes;
  2898. size_t explicitBlockIndexBytes;
  2899. friend class ConcurrentQueue;
  2900. private:
  2901. static MemStats getFor(ConcurrentQueue* q)
  2902. {
  2903. MemStats stats = { 0 };
  2904. stats.elementsEnqueued = q->size_approx();
  2905. auto block = q->freeList.head_unsafe();
  2906. while (block != nullptr) {
  2907. ++stats.allocatedBlocks;
  2908. ++stats.freeBlocks;
  2909. block = block->freeListNext.load(std::memory_order_relaxed);
  2910. }
  2911. for (auto ptr = q->producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) {
  2912. bool implicit = dynamic_cast<ImplicitProducer*>(ptr) != nullptr;
  2913. stats.implicitProducers += implicit ? 1 : 0;
  2914. stats.explicitProducers += implicit ? 0 : 1;
  2915. if (implicit) {
  2916. auto prod = static_cast<ImplicitProducer*>(ptr);
  2917. stats.queueClassBytes += sizeof(ImplicitProducer);
  2918. auto head = prod->headIndex.load(std::memory_order_relaxed);
  2919. auto tail = prod->tailIndex.load(std::memory_order_relaxed);
  2920. auto hash = prod->blockIndex.load(std::memory_order_relaxed);
  2921. if (hash != nullptr) {
  2922. for (size_t i = 0; i != hash->capacity; ++i) {
  2923. if (hash->index[i]->key.load(std::memory_order_relaxed) != ImplicitProducer::INVALID_BLOCK_BASE && hash->index[i]->value.load(std::memory_order_relaxed) != nullptr) {
  2924. ++stats.allocatedBlocks;
  2925. ++stats.ownedBlocksImplicit;
  2926. }
  2927. }
  2928. stats.implicitBlockIndexBytes += hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry);
  2929. for (; hash != nullptr; hash = hash->prev) {
  2930. stats.implicitBlockIndexBytes += sizeof(typename ImplicitProducer::BlockIndexHeader) + hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry*);
  2931. }
  2932. }
  2933. for (; details::circular_less_than<index_t>(head, tail); head += BLOCK_SIZE) {
  2934. //auto block = prod->get_block_index_entry_for_index(head);
  2935. ++stats.usedBlocks;
  2936. }
  2937. }
  2938. else {
  2939. auto prod = static_cast<ExplicitProducer*>(ptr);
  2940. stats.queueClassBytes += sizeof(ExplicitProducer);
  2941. auto tailBlock = prod->tailBlock;
  2942. bool wasNonEmpty = false;
  2943. if (tailBlock != nullptr) {
  2944. auto block = tailBlock;
  2945. do {
  2946. ++stats.allocatedBlocks;
  2947. if (!block->ConcurrentQueue::Block::template is_empty<explicit_context>() || wasNonEmpty) {
  2948. ++stats.usedBlocks;
  2949. wasNonEmpty = wasNonEmpty || block != tailBlock;
  2950. }
  2951. ++stats.ownedBlocksExplicit;
  2952. block = block->next;
  2953. } while (block != tailBlock);
  2954. }
  2955. auto index = prod->blockIndex.load(std::memory_order_relaxed);
  2956. while (index != nullptr) {
  2957. stats.explicitBlockIndexBytes += sizeof(typename ExplicitProducer::BlockIndexHeader) + index->size * sizeof(typename ExplicitProducer::BlockIndexEntry);
  2958. index = static_cast<typename ExplicitProducer::BlockIndexHeader*>(index->prev);
  2959. }
  2960. }
  2961. }
  2962. auto freeOnInitialPool = q->initialBlockPoolIndex.load(std::memory_order_relaxed) >= q->initialBlockPoolSize ? 0 : q->initialBlockPoolSize - q->initialBlockPoolIndex.load(std::memory_order_relaxed);
  2963. stats.allocatedBlocks += freeOnInitialPool;
  2964. stats.freeBlocks += freeOnInitialPool;
  2965. stats.blockClassBytes = sizeof(Block) * stats.allocatedBlocks;
  2966. stats.queueClassBytes += sizeof(ConcurrentQueue);
  2967. return stats;
  2968. }
  2969. };
  2970. // For debugging only. Not thread-safe.
  2971. MemStats getMemStats()
  2972. {
  2973. return MemStats::getFor(this);
  2974. }
  2975. private:
  2976. friend struct MemStats;
  2977. #endif
  2978. //////////////////////////////////
  2979. // Producer list manipulation
  2980. //////////////////////////////////
  2981. ProducerBase *recycle_or_create_producer(bool isExplicit) {
  2982. bool recycled;
  2983. return recycle_or_create_producer(isExplicit, recycled);
  2984. }
  2985. ProducerBase *recycle_or_create_producer(bool isExplicit, bool &recycled) {
  2986. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
  2987. debug::DebugLock lock(implicitProdMutex);
  2988. #endif
  2989. // Try to re-use one first
  2990. for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) {
  2991. if (ptr->inactive.load(std::memory_order_relaxed) && ptr->isExplicit == isExplicit) {
  2992. bool expected = true;
  2993. if (ptr->inactive.compare_exchange_strong(expected, /* desired */ false, std::memory_order_acquire,
  2994. std::memory_order_relaxed)) {
  2995. // We caught one! It's been marked as activated, the caller can have it
  2996. recycled = true;
  2997. return ptr;
  2998. }
  2999. }
  3000. }
  3001. recycled = false;
  3002. return add_producer(
  3003. isExplicit ? static_cast<ProducerBase *>(create<ExplicitProducer>(this)) : create<ImplicitProducer>(
  3004. this));
  3005. }
  3006. ProducerBase *add_producer(ProducerBase *producer) {
  3007. // Handle failed memory allocation
  3008. if (producer == nullptr) {
  3009. return nullptr;
  3010. }
  3011. producerCount.fetch_add(1, std::memory_order_relaxed);
  3012. // Add it to the lock-free list
  3013. auto prevTail = producerListTail.load(std::memory_order_relaxed);
  3014. do {
  3015. producer->next = prevTail;
  3016. } while (!producerListTail.compare_exchange_weak(prevTail, producer, std::memory_order_release,
  3017. std::memory_order_relaxed));
  3018. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  3019. if (producer->isExplicit) {
  3020. auto prevTailExplicit = explicitProducers.load(std::memory_order_relaxed);
  3021. do {
  3022. static_cast<ExplicitProducer*>(producer)->nextExplicitProducer = prevTailExplicit;
  3023. } while (!explicitProducers.compare_exchange_weak(prevTailExplicit, static_cast<ExplicitProducer*>(producer), std::memory_order_release, std::memory_order_relaxed));
  3024. }
  3025. else {
  3026. auto prevTailImplicit = implicitProducers.load(std::memory_order_relaxed);
  3027. do {
  3028. static_cast<ImplicitProducer*>(producer)->nextImplicitProducer = prevTailImplicit;
  3029. } while (!implicitProducers.compare_exchange_weak(prevTailImplicit, static_cast<ImplicitProducer*>(producer), std::memory_order_release, std::memory_order_relaxed));
  3030. }
  3031. #endif
  3032. return producer;
  3033. }
  3034. void reown_producers() {
  3035. // After another instance is moved-into/swapped-with this one, all the
  3036. // producers we stole still think their parents are the other queue.
  3037. // So fix them up!
  3038. for (auto ptr = producerListTail.load(std::memory_order_relaxed); ptr != nullptr; ptr = ptr->next_prod()) {
  3039. ptr->parent = this;
  3040. }
  3041. }
  3042. //////////////////////////////////
  3043. // Implicit producer hash
  3044. //////////////////////////////////
  3045. struct ImplicitProducerKVP {
  3046. std::atomic<details::thread_id_t> key;
  3047. ImplicitProducer *value; // No need for atomicity since it's only read by the thread that sets it in the first place
  3048. ImplicitProducerKVP() : value(nullptr) {}
  3049. ImplicitProducerKVP(ImplicitProducerKVP &&other)
  3050. MOODYCAMEL_NOEXCEPT
  3051. {
  3052. key.store(other.key.load(std::memory_order_relaxed), std::memory_order_relaxed);
  3053. value = other.value;
  3054. }
  3055. inline ImplicitProducerKVP &operator=(ImplicitProducerKVP &&other)
  3056. MOODYCAMEL_NOEXCEPT
  3057. {
  3058. swap(other);
  3059. return *this;
  3060. }
  3061. inline void swap(ImplicitProducerKVP &other)
  3062. MOODYCAMEL_NOEXCEPT
  3063. {
  3064. if (this != &other) {
  3065. details::swap_relaxed(key, other.key);
  3066. std::swap(value, other.value);
  3067. }
  3068. }
  3069. };
  3070. template<typename XT, typename XTraits>
  3071. friend void moodycamel::swap(typename ConcurrentQueue<XT, XTraits>::ImplicitProducerKVP &,
  3072. typename ConcurrentQueue<XT, XTraits>::ImplicitProducerKVP &)
  3073. MOODYCAMEL_NOEXCEPT;
  3074. struct ImplicitProducerHash {
  3075. size_t capacity;
  3076. ImplicitProducerKVP *entries;
  3077. ImplicitProducerHash *prev;
  3078. };
  3079. inline void populate_initial_implicit_producer_hash() {
  3080. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) {
  3081. return;
  3082. } else {
  3083. implicitProducerHashCount.store(0, std::memory_order_relaxed);
  3084. auto hash = &initialImplicitProducerHash;
  3085. hash->capacity = INITIAL_IMPLICIT_PRODUCER_HASH_SIZE;
  3086. hash->entries = &initialImplicitProducerHashEntries[0];
  3087. for (size_t i = 0; i != INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; ++i) {
  3088. initialImplicitProducerHashEntries[i].key.store(details::invalid_thread_id,
  3089. std::memory_order_relaxed);
  3090. }
  3091. hash->prev = nullptr;
  3092. implicitProducerHash.store(hash, std::memory_order_relaxed);
  3093. }
  3094. }
  3095. void swap_implicit_producer_hashes(ConcurrentQueue &other) {
  3096. MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) {
  3097. return;
  3098. } else {
  3099. // Swap (assumes our implicit producer hash is initialized)
  3100. initialImplicitProducerHashEntries.swap(other.initialImplicitProducerHashEntries);
  3101. initialImplicitProducerHash.entries = &initialImplicitProducerHashEntries[0];
  3102. other.initialImplicitProducerHash.entries = &other.initialImplicitProducerHashEntries[0];
  3103. details::swap_relaxed(implicitProducerHashCount, other.implicitProducerHashCount);
  3104. details::swap_relaxed(implicitProducerHash, other.implicitProducerHash);
  3105. if (implicitProducerHash.load(std::memory_order_relaxed) == &other.initialImplicitProducerHash) {
  3106. implicitProducerHash.store(&initialImplicitProducerHash, std::memory_order_relaxed);
  3107. } else {
  3108. ImplicitProducerHash *hash;
  3109. for (hash = implicitProducerHash.load(std::memory_order_relaxed);
  3110. hash->prev != &other.initialImplicitProducerHash; hash = hash->prev) {
  3111. continue;
  3112. }
  3113. hash->prev = &initialImplicitProducerHash;
  3114. }
  3115. if (other.implicitProducerHash.load(std::memory_order_relaxed) == &initialImplicitProducerHash) {
  3116. other.implicitProducerHash.store(&other.initialImplicitProducerHash, std::memory_order_relaxed);
  3117. } else {
  3118. ImplicitProducerHash *hash;
  3119. for (hash = other.implicitProducerHash.load(std::memory_order_relaxed);
  3120. hash->prev != &initialImplicitProducerHash; hash = hash->prev) {
  3121. continue;
  3122. }
  3123. hash->prev = &other.initialImplicitProducerHash;
  3124. }
  3125. }
  3126. }
  3127. // Only fails (returns nullptr) if memory allocation fails
  3128. ImplicitProducer *get_or_add_implicit_producer() {
  3129. // Note that since the data is essentially thread-local (key is thread ID),
  3130. // there's a reduced need for fences (memory ordering is already consistent
  3131. // for any individual thread), except for the current table itself.
  3132. // Start by looking for the thread ID in the current and all previous hash tables.
  3133. // If it's not found, it must not be in there yet, since this same thread would
  3134. // have added it previously to one of the tables that we traversed.
  3135. // Code and algorithm adapted from http://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table
  3136. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
  3137. debug::DebugLock lock(implicitProdMutex);
  3138. #endif
  3139. auto id = details::thread_id();
  3140. auto hashedId = details::hash_thread_id(id);
  3141. auto mainHash = implicitProducerHash.load(std::memory_order_acquire);
  3142. assert(mainHash != nullptr); // silence clang-tidy and MSVC warnings (hash cannot be null)
  3143. for (auto hash = mainHash; hash != nullptr; hash = hash->prev) {
  3144. // Look for the id in this hash
  3145. auto index = hashedId;
  3146. while (true) { // Not an infinite loop because at least one slot is free in the hash table
  3147. index &= hash->capacity - 1;
  3148. auto probedKey = hash->entries[index].key.load(std::memory_order_relaxed);
  3149. if (probedKey == id) {
  3150. // Found it! If we had to search several hashes deep, though, we should lazily add it
  3151. // to the current main hash table to avoid the extended search next time.
  3152. // Note there's guaranteed to be room in the current hash table since every subsequent
  3153. // table implicitly reserves space for all previous tables (there's only one
  3154. // implicitProducerHashCount).
  3155. auto value = hash->entries[index].value;
  3156. if (hash != mainHash) {
  3157. index = hashedId;
  3158. while (true) {
  3159. index &= mainHash->capacity - 1;
  3160. probedKey = mainHash->entries[index].key.load(std::memory_order_relaxed);
  3161. auto empty = details::invalid_thread_id;
  3162. #ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  3163. auto reusable = details::invalid_thread_id2;
  3164. if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_relaxed, std::memory_order_relaxed)) ||
  3165. (probedKey == reusable && mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_acquire, std::memory_order_acquire))) {
  3166. #else
  3167. if ((probedKey == empty &&
  3168. mainHash->entries[index].key.compare_exchange_strong(empty, id,
  3169. std::memory_order_relaxed,
  3170. std::memory_order_relaxed))) {
  3171. #endif
  3172. mainHash->entries[index].value = value;
  3173. break;
  3174. }
  3175. ++index;
  3176. }
  3177. }
  3178. return value;
  3179. }
  3180. if (probedKey == details::invalid_thread_id) {
  3181. break; // Not in this hash table
  3182. }
  3183. ++index;
  3184. }
  3185. }
  3186. // Insert!
  3187. auto newCount = 1 + implicitProducerHashCount.fetch_add(1, std::memory_order_relaxed);
  3188. while (true) {
  3189. // NOLINTNEXTLINE(clang-analyzer-core.NullDereference)
  3190. if (newCount >= (mainHash->capacity >> 1) &&
  3191. !implicitProducerHashResizeInProgress.test_and_set(std::memory_order_acquire)) {
  3192. // We've acquired the resize lock, try to allocate a bigger hash table.
  3193. // Note the acquire fence synchronizes with the release fence at the end of this block, and hence when
  3194. // we reload implicitProducerHash it must be the most recent version (it only gets changed within this
  3195. // locked block).
  3196. mainHash = implicitProducerHash.load(std::memory_order_acquire);
  3197. if (newCount >= (mainHash->capacity >> 1)) {
  3198. auto newCapacity = mainHash->capacity << 1;
  3199. while (newCount >= (newCapacity >> 1)) {
  3200. newCapacity <<= 1;
  3201. }
  3202. auto raw = static_cast<char *>((Traits::malloc)(
  3203. sizeof(ImplicitProducerHash) + std::alignment_of<ImplicitProducerKVP>::value - 1 +
  3204. sizeof(ImplicitProducerKVP) * newCapacity));
  3205. if (raw == nullptr) {
  3206. // Allocation failed
  3207. implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed);
  3208. implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed);
  3209. return nullptr;
  3210. }
  3211. auto newHash = new(raw) ImplicitProducerHash;
  3212. newHash->capacity = static_cast<size_t>(newCapacity);
  3213. newHash->entries = reinterpret_cast<ImplicitProducerKVP *>(details::align_for<ImplicitProducerKVP>(
  3214. raw + sizeof(ImplicitProducerHash)));
  3215. for (size_t i = 0; i != newCapacity; ++i) {
  3216. new(newHash->entries + i) ImplicitProducerKVP;
  3217. newHash->entries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed);
  3218. }
  3219. newHash->prev = mainHash;
  3220. implicitProducerHash.store(newHash, std::memory_order_release);
  3221. implicitProducerHashResizeInProgress.clear(std::memory_order_release);
  3222. mainHash = newHash;
  3223. } else {
  3224. implicitProducerHashResizeInProgress.clear(std::memory_order_release);
  3225. }
  3226. }
  3227. // If it's < three-quarters full, add to the old one anyway so that we don't have to wait for the next table
  3228. // to finish being allocated by another thread (and if we just finished allocating above, the condition will
  3229. // always be true)
  3230. if (newCount < (mainHash->capacity >> 1) + (mainHash->capacity >> 2)) {
  3231. bool recycled;
  3232. auto producer = static_cast<ImplicitProducer *>(recycle_or_create_producer(false, recycled));
  3233. if (producer == nullptr) {
  3234. implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed);
  3235. return nullptr;
  3236. }
  3237. if (recycled) {
  3238. implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed);
  3239. }
  3240. #ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  3241. producer->threadExitListener.callback = &ConcurrentQueue::implicit_producer_thread_exited_callback;
  3242. producer->threadExitListener.userData = producer;
  3243. details::ThreadExitNotifier::subscribe(&producer->threadExitListener);
  3244. #endif
  3245. auto index = hashedId;
  3246. while (true) {
  3247. index &= mainHash->capacity - 1;
  3248. auto probedKey = mainHash->entries[index].key.load(std::memory_order_relaxed);
  3249. auto empty = details::invalid_thread_id;
  3250. #ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  3251. auto reusable = details::invalid_thread_id2;
  3252. if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_relaxed, std::memory_order_relaxed)) ||
  3253. (probedKey == reusable && mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_acquire, std::memory_order_acquire))) {
  3254. #else
  3255. if ((probedKey == empty &&
  3256. mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_relaxed,
  3257. std::memory_order_relaxed))) {
  3258. #endif
  3259. mainHash->entries[index].value = producer;
  3260. break;
  3261. }
  3262. ++index;
  3263. }
  3264. return producer;
  3265. }
  3266. // Hmm, the old hash is quite full and somebody else is busy allocating a new one.
  3267. // We need to wait for the allocating thread to finish (if it succeeds, we add, if not,
  3268. // we try to allocate ourselves).
  3269. mainHash = implicitProducerHash.load(std::memory_order_acquire);
  3270. }
  3271. }
  3272. #ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
  3273. void implicit_producer_thread_exited(ImplicitProducer* producer)
  3274. {
  3275. // Remove from thread exit listeners
  3276. details::ThreadExitNotifier::unsubscribe(&producer->threadExitListener);
  3277. // Remove from hash
  3278. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
  3279. debug::DebugLock lock(implicitProdMutex);
  3280. #endif
  3281. auto hash = implicitProducerHash.load(std::memory_order_acquire);
  3282. assert(hash != nullptr); // The thread exit listener is only registered if we were added to a hash in the first place
  3283. auto id = details::thread_id();
  3284. auto hashedId = details::hash_thread_id(id);
  3285. details::thread_id_t probedKey;
  3286. // We need to traverse all the hashes just in case other threads aren't on the current one yet and are
  3287. // trying to add an entry thinking there's a free slot (because they reused a producer)
  3288. for (; hash != nullptr; hash = hash->prev) {
  3289. auto index = hashedId;
  3290. do {
  3291. index &= hash->capacity - 1;
  3292. probedKey = hash->entries[index].key.load(std::memory_order_relaxed);
  3293. if (probedKey == id) {
  3294. hash->entries[index].key.store(details::invalid_thread_id2, std::memory_order_release);
  3295. break;
  3296. }
  3297. ++index;
  3298. } while (probedKey != details::invalid_thread_id); // Can happen if the hash has changed but we weren't put back in it yet, or if we weren't added to this hash in the first place
  3299. }
  3300. // Mark the queue as being recyclable
  3301. producer->inactive.store(true, std::memory_order_release);
  3302. }
  3303. static void implicit_producer_thread_exited_callback(void* userData)
  3304. {
  3305. auto producer = static_cast<ImplicitProducer*>(userData);
  3306. auto queue = producer->parent;
  3307. queue->implicit_producer_thread_exited(producer);
  3308. }
  3309. #endif
  3310. //////////////////////////////////
  3311. // Utility functions
  3312. //////////////////////////////////
  3313. template<typename TAlign>
  3314. static inline void *aligned_malloc(size_t size) {
  3315. MOODYCAMEL_CONSTEXPR_IF (std::alignment_of<TAlign>::value <=
  3316. std::alignment_of<details::max_align_t>::value) return (Traits::malloc)(size);
  3317. else {
  3318. size_t alignment = std::alignment_of<TAlign>::value;
  3319. void *raw = (Traits::malloc)(size + alignment - 1 + sizeof(void *));
  3320. if (!raw)
  3321. return nullptr;
  3322. char *ptr = details::align_for<TAlign>(reinterpret_cast<char *>(raw) + sizeof(void *));
  3323. *(reinterpret_cast<void **>(ptr) - 1) = raw;
  3324. return ptr;
  3325. }
  3326. }
  3327. template<typename TAlign>
  3328. static inline void aligned_free(void *ptr) {
  3329. MOODYCAMEL_CONSTEXPR_IF (std::alignment_of<TAlign>::value <=
  3330. std::alignment_of<details::max_align_t>::value) return (Traits::free)(ptr);
  3331. else
  3332. (Traits::free)(ptr ? *(reinterpret_cast<void **>(ptr) - 1) : nullptr);
  3333. }
  3334. template<typename U>
  3335. static inline U *create_array(size_t count) {
  3336. assert(count > 0);
  3337. U *p = static_cast<U *>(aligned_malloc<U>(sizeof(U) * count));
  3338. if (p == nullptr)
  3339. return nullptr;
  3340. for (size_t i = 0; i != count; ++i)
  3341. new(p + i) U();
  3342. return p;
  3343. }
  3344. template<typename U>
  3345. static inline void destroy_array(U *p, size_t count) {
  3346. if (p != nullptr) {
  3347. assert(count > 0);
  3348. for (size_t i = count; i != 0;)
  3349. (p + --i)->~U();
  3350. }
  3351. aligned_free<U>(p);
  3352. }
  3353. template<typename U>
  3354. static inline U *create() {
  3355. void *p = aligned_malloc<U>(sizeof(U));
  3356. return p != nullptr ? new(p) U : nullptr;
  3357. }
  3358. template<typename U, typename A1>
  3359. static inline U *create(A1 &&a1) {
  3360. void *p = aligned_malloc<U>(sizeof(U));
  3361. return p != nullptr ? new(p) U(std::forward<A1>(a1)) : nullptr;
  3362. }
  3363. template<typename U>
  3364. static inline void destroy(U *p) {
  3365. if (p != nullptr)
  3366. p->~U();
  3367. aligned_free<U>(p);
  3368. }
  3369. private:
  3370. std::atomic<ProducerBase *> producerListTail;
  3371. std::atomic<std::uint32_t> producerCount;
  3372. std::atomic<size_t> initialBlockPoolIndex;
  3373. Block *initialBlockPool;
  3374. size_t initialBlockPoolSize;
  3375. #ifndef MCDBGQ_USEDEBUGFREELIST
  3376. FreeList<Block> freeList;
  3377. #else
  3378. debug::DebugFreeList<Block> freeList;
  3379. #endif
  3380. std::atomic<ImplicitProducerHash *> implicitProducerHash;
  3381. std::atomic<size_t> implicitProducerHashCount; // Number of slots logically used
  3382. ImplicitProducerHash initialImplicitProducerHash;
  3383. std::array<ImplicitProducerKVP, INITIAL_IMPLICIT_PRODUCER_HASH_SIZE> initialImplicitProducerHashEntries;
  3384. std::atomic_flag implicitProducerHashResizeInProgress;
  3385. std::atomic<std::uint32_t> nextExplicitConsumerId;
  3386. std::atomic<std::uint32_t> globalExplicitConsumerOffset;
  3387. #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
  3388. debug::DebugMutex implicitProdMutex;
  3389. #endif
  3390. #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
  3391. std::atomic<ExplicitProducer*> explicitProducers;
  3392. std::atomic<ImplicitProducer*> implicitProducers;
  3393. #endif
  3394. };
  3395. template<typename T, typename Traits>
  3396. ProducerToken::ProducerToken(ConcurrentQueue<T, Traits> &queue)
  3397. : producer(queue.recycle_or_create_producer(true)) {
  3398. if (producer != nullptr) {
  3399. producer->token = this;
  3400. }
  3401. }
  3402. template<typename T, typename Traits>
  3403. ProducerToken::ProducerToken(BlockingConcurrentQueue<T, Traits> &queue)
  3404. : producer(reinterpret_cast<ConcurrentQueue<T, Traits> *>(&queue)->recycle_or_create_producer(true)) {
  3405. if (producer != nullptr) {
  3406. producer->token = this;
  3407. }
  3408. }
  3409. template<typename T, typename Traits>
  3410. ConsumerToken::ConsumerToken(ConcurrentQueue<T, Traits> &queue)
  3411. : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) {
  3412. initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release);
  3413. lastKnownGlobalOffset = static_cast<std::uint32_t>(-1);
  3414. }
  3415. template<typename T, typename Traits>
  3416. ConsumerToken::ConsumerToken(BlockingConcurrentQueue<T, Traits> &queue)
  3417. : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) {
  3418. initialOffset = reinterpret_cast<ConcurrentQueue<T, Traits> *>(&queue)->nextExplicitConsumerId.fetch_add(1,
  3419. std::memory_order_release);
  3420. lastKnownGlobalOffset = static_cast<std::uint32_t>(-1);
  3421. }
  3422. template<typename T, typename Traits>
  3423. inline void swap(ConcurrentQueue<T, Traits> &a, ConcurrentQueue<T, Traits> &b)
  3424. MOODYCAMEL_NOEXCEPT {
  3425. a.
  3426. swap(b);
  3427. }
  3428. inline void swap(ProducerToken &a, ProducerToken &b)
  3429. MOODYCAMEL_NOEXCEPT
  3430. {
  3431. a.
  3432. swap(b);
  3433. }
  3434. inline void swap(ConsumerToken &a, ConsumerToken &b)
  3435. MOODYCAMEL_NOEXCEPT
  3436. {
  3437. a.
  3438. swap(b);
  3439. }
  3440. template<typename T, typename Traits>
  3441. inline void swap(typename ConcurrentQueue<T, Traits>::ImplicitProducerKVP &a,
  3442. typename ConcurrentQueue<T, Traits>::ImplicitProducerKVP &b)
  3443. MOODYCAMEL_NOEXCEPT
  3444. {
  3445. a.
  3446. swap(b);
  3447. }
  3448. }
  3449. #if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17)
  3450. #pragma warning(pop)
  3451. #endif
  3452. #if defined(__GNUC__) && !defined(__INTEL_COMPILER)
  3453. #pragma GCC diagnostic pop
  3454. #endif