post() / post_bulk() inlining improvements (#141)

Author: tzcnt

include/tmc/detail/ex_cpu.ipp

@@ -367,8 +367,8 @@ void ex_cpu::post(work_item&& Item, size_t Priority, size_t ThreadHint) {
     if (!fromExecThread || ThreadHint != tmc::current_thread_index()) {
       notify_n(1, Priority, ThreadHint, fromExecThread, true);
     }
-  } else {
-    if (fromExecThread) {
+  } else [[likely]] {
+    if (fromExecThread) [[likely]] {
       work_queues[Priority].enqueue_ex_cpu(
         static_cast<work_item&&>(Item), Priority
       );

include/tmc/detail/qu_lockfree.hpp

@@ -727,7 +727,7 @@ class ConcurrentQueue {
   }
 
   // Enqueues a single item using this ex_cpu thread's explicit producer.
-  template <typename U> inline void enqueue_ex_cpu(U&& item, size_t priority) {
+  template <typename U> TMC_FORCE_INLINE inline void enqueue_ex_cpu(U&& item, size_t priority) {
     ExplicitProducer** producers =
       static_cast<ExplicitProducer**>(tmc::detail::this_thread::producers);
     ExplicitProducer* this_thread_prod =
@@ -744,7 +744,7 @@ class ConcurrentQueue {
 
   // Enqueues several items using this ex_cpu thread's explicit producer.
   template <typename It>
-  void enqueue_bulk_ex_cpu(It itemFirst, size_t count, size_t priority) {
+  TMC_FORCE_INLINE void enqueue_bulk_ex_cpu(It itemFirst, size_t count, size_t priority) {
     ExplicitProducer** producers =
       static_cast<ExplicitProducer**>(tmc::detail::this_thread::producers);
     ExplicitProducer* this_thread_prod =
@@ -1416,7 +1416,7 @@ class ConcurrentQueue {
       }
     }
 
-    template <typename U> inline void enqueue(U&& element) {
+    template <typename U> TMC_FORCE_INLINE inline void enqueue(U&& element) {
       index_t currentTailIndex =
         this->tailIndex.load(std::memory_order_relaxed);
       index_t newTailIndex = 1 + currentTailIndex;
@@ -1748,7 +1748,7 @@ class ConcurrentQueue {
     }
 
     template <typename It>
-    void MOODYCAMEL_NO_TSAN enqueue_bulk(It itemFirst, size_t count) {
+    TMC_FORCE_INLINE void MOODYCAMEL_NO_TSAN enqueue_bulk(It itemFirst, size_t count) {
       // static constexpr bool HasMoveConstructor = std::is_constructible_v<
       //   T, std::add_rvalue_reference_t<std::iter_value_t<It>>>;
       static constexpr bool HasNoexceptMoveConstructor =

include/tmc/ex_cpu.hpp

@@ -222,7 +222,7 @@ class ex_cpu {
     if (!fromExecThread) {
       ++ref_count;
     }
-    if (ThreadHint < thread_count()) {
+    if (ThreadHint < thread_count()) [[unlikely]] {
       size_t enqueuedCount = thread_states[ThreadHint].inbox->try_push_bulk(
         static_cast<It&&>(Items), Count, Priority
       );
@@ -233,8 +233,8 @@ class ex_cpu {
         }
       }
     }
-    if (Count > 0) {
-      if (fromExecThread) {
+    if (Count > 0) [[likely]] {
+      if (fromExecThread) [[likely]] {
         work_queues[Priority].enqueue_bulk_ex_cpu(
           static_cast<It&&>(Items), Count, Priority
         );