diff --git a/include/psi/functionoid/detail/callable_base.hpp b/include/psi/functionoid/detail/callable_base.hpp index b8f908b..04818b1 100644 --- a/include/psi/functionoid/detail/callable_base.hpp +++ b/include/psi/functionoid/detail/callable_base.hpp @@ -846,7 +846,7 @@ class callable_base : public callable_tag empty_handler_traits::allowsSmallObjectOptimization ); empty_handler_manager::assign( EmptyHandler(), p_function_->functor_, std::allocator() ); - p_function_->store_vtable( &empty_handler_vtable_ ); + p_function_->store_vtable( empty_handler_vtable_ ); } } @@ -881,7 +881,7 @@ class callable_base : public callable_tag callable_base( no_eh_state_construction_trick_tag, Constructor const constructor, Args && ... args ) noexcept( noexcept( constructor( std::declval(), std::forward( args )... ) ) ) { auto const & vtable( constructor( *this, std::forward( args )... ) ); - BOOST_ASSUME( load_vtable( std::memory_order_relaxed ) == &vtable ); + BOOST_ASSUME( vtable_slot( std::memory_order_relaxed ) == &vtable ); } // destructor = trivial promises no target ever needs destroying (and the @@ -900,10 +900,11 @@ class callable_base : public callable_tag /// Engagement probe for a reader that may race the thread which assigns to /// this callable. Only the vtable pointer load is ordered - what it orders /// is the target buffer written before it (see store_vtable()). + /// Always atomic, whatever Traits::concurrent_reads says: an explicitly + /// ordered read is the caller opting in for this call. bool empty( void const * const p_empty_handler_vtable, std::memory_order const order ) const noexcept - requires ( Traits::concurrent_reads ) { - auto const p_vtable{ load_vtable( order ) }; + auto const p_vtable{ atomic_load_vtable( order ) }; BOOST_ASSUME( p_vtable ); return p_vtable->is_empty_handler_vtable( p_empty_handler_vtable ); } @@ -934,26 +935,62 @@ class callable_base : public callable_tag /// entire API surface, a single memory_order overload does not) /// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0045r0.pdf /// (08.11.2016.) (Domagoj Saric) - vtable const * load_vtable( std::memory_order const order ) const noexcept + /// Unconditionally atomic accessors - the per-call opt-in. A caller that + /// asks for an ordering gets it whether or not the Traits opted in, which + /// is what makes a single publish-once site usable without taxing every + /// other instantiation of the type (see policies.hpp). + /// The const_cast strips the member's cv/__restrict qualification: + /// std::atomic_ref binds to neither, and both concern the *pointee* (the + /// vtable) rather than this pointer's own storage, so nothing they promise + /// is being broken. It is also what lets the member itself stay exactly as + /// it always was - no conditional type, no `mutable`. + /// ⚠ The caller owns the contract std::atomic_ref itself imposes: while an + /// ordered access is in flight, every *conflicting* access to this object + /// must also be ordered. Traits::concurrent_reads is the way to have that + /// hold by construction. + vtable const * atomic_load_vtable( std::memory_order const order ) const noexcept { - if constexpr ( Traits::concurrent_reads ) { return std::atomic_ref{ p_vtable_ }.load( order ); } + return std::atomic_ref{ *const_cast( &p_vtable_ ) }.load( order ); + } + void atomic_store_vtable( vtable const & vt, std::memory_order const order ) noexcept + { + std::atomic_ref{ *const_cast( &p_vtable_ ) }.store( &vt, order ); + } + + /// The raw slot read: what the pointer currently HOLDS, which mid + /// construction is legitimately not a vtable at all (debug_clear's + /// invalid_ptr) - hence a pointer, and hence separate from load_vtable(). + /// The identity/validity assertions are its only callers. + vtable const * vtable_slot( std::memory_order const order ) const noexcept + { + if constexpr ( Traits::concurrent_reads ) { return atomic_load_vtable( order ); } else { return p_vtable_; } } + vtable const & load_vtable( std::memory_order const order ) const noexcept + { + auto const p_vtable{ vtable_slot( order ) }; + BOOST_ASSUME( p_vtable ); + return *p_vtable; + } + /// The publishing store. Every assignment path writes the target buffer /// _first_ and the vtable pointer last, so this release store is exactly /// the edge a reader's acquire load of it needs: observing a non-empty /// vtable implies the target it describes is fully constructed. - /// Deliberately not exposed with a memory_order parameter: a weaker - /// publish would defeat the mechanism, so there is no legitimate choice for - /// a caller to make - only a way to get it wrong. - void store_vtable( vtable const * const p_vtable ) noexcept + /// `order` unset (the default) means "let the Traits decide" - a release + /// store for concurrent_reads, a plain one otherwise. Set, it is the + /// per-call opt-in and always publishes atomically. There is deliberately + /// no way to ask for something *weaker* than the Traits already give: a + /// weaker publish only defeats the mechanism. + void store_vtable( vtable const & vt, publish_order const order = {} ) noexcept { - if constexpr ( Traits::concurrent_reads ) { std::atomic_ref{ p_vtable_ }.store( p_vtable, std::memory_order_release ); } - else { p_vtable_ = p_vtable; } + if ( order ) { atomic_store_vtable( vt, *order ); return; } + if constexpr ( Traits::concurrent_reads ) { atomic_store_vtable( vt, std::memory_order_release ); } + else { p_vtable_ = &vt; } } - auto const & get_vtable() const noexcept { auto const p_vtable{ load_vtable( std::memory_order_relaxed ) }; BOOST_ASSUME( p_vtable ); return *p_vtable; } + auto const & get_vtable() const noexcept { return load_vtable( std::memory_order_relaxed ); } buffer & functor() const noexcept { return functor_; } @@ -987,24 +1024,25 @@ class callable_base : public callable_tag [[ maybe_unused ]] vtable const & functor_vtable, vtable const & empty_handler_vtable, - Allocator + Allocator, + publish_order const publish = {} ) { auto const same_traits{ std::is_convertible_v *, callable_base const *> }; if constexpr ( same_traits ) { - BOOST_ASSUME( &functor_vtable == f.load_vtable( std::memory_order_relaxed ) ); + BOOST_ASSUME( &functor_vtable == f.vtable_slot( std::memory_order_relaxed ) ); } if constexpr ( direct ) { BOOST_ASSUME( &f != static_cast( this ) ); BOOST_ASSERT ( - ( this->load_vtable( std::memory_order_relaxed ) == &empty_handler_vtable ) || + ( this->vtable_slot( std::memory_order_relaxed ) == &empty_handler_vtable ) || // just being constructed/inside a no_eh_state_construction_trick constructor in a debug build: - ( this->load_vtable( std::memory_order_relaxed ) == invalid_ptr ) + ( this->vtable_slot( std::memory_order_relaxed ) == invalid_ptr ) ); - assign_functionoid_direct( std::forward( f ), empty_handler_vtable ); + assign_functionoid_direct( std::forward( f ), empty_handler_vtable, publish ); } else { @@ -1012,7 +1050,7 @@ class callable_base : public callable_tag if constexpr ( same_traits && ( Traits::destructor != support_level::trivial ) ) if ( BOOST_UNLIKELY( &f == this ) ) return; - assign_functionoid_guarded( std::forward( f ), empty_handler_vtable ); + assign_functionoid_guarded( std::forward( f ), empty_handler_vtable, publish ); } } @@ -1028,7 +1066,8 @@ class callable_base : public callable_tag FunctionObj && f, vtable const & functor_vtable, vtable const & empty_handler_vtable, - Allocator const a + Allocator const a, + publish_order const publish = {} ) { using tag = typename get_function_tag::type; @@ -1042,10 +1081,10 @@ class callable_base : public callable_tag // functionoid.hpp as to why a null vtable is allowed and expected // here. // (02.11.2010.) (Domagoj Saric) - BOOST_ASSERT( this->load_vtable( std::memory_order_relaxed ) == &empty_handler_vtable || /*just being constructed/inside a no_eh_state_construction_trick constructor in a debug build:*/ this->load_vtable( std::memory_order_relaxed ) == invalid_ptr ); + BOOST_ASSERT( this->vtable_slot( std::memory_order_relaxed ) == &empty_handler_vtable || /*just being constructed/inside a no_eh_state_construction_trick constructor in a debug build:*/ this->vtable_slot( std::memory_order_relaxed ) == invalid_ptr ); using functor_manager = detail::functor_manager, Allocator, buffer>; functor_manager::assign( std::forward( f ), this->functor_, a ); - this->store_vtable( &functor_vtable ); + this->store_vtable( functor_vtable, publish ); } else { @@ -1054,7 +1093,8 @@ class callable_base : public callable_tag std::forward( f ), functor_vtable, empty_handler_vtable, - a + a, + publish ); } } @@ -1082,13 +1122,14 @@ class callable_base : public callable_tag F && f, vtable const & functor_vtable, vtable const & /*empty_handler_vtable*/, - Allocator const a + Allocator const a, + publish_order const publish = {} ) noexcept { using functor_manager = functor_manager; this->destroy(); functor_manager::assign( std::forward( f ), this->functor_, a ); - this->store_vtable( &functor_vtable ); + this->store_vtable( functor_vtable, publish ); } template @@ -1098,32 +1139,43 @@ class callable_base : public callable_tag F && f, vtable const & functor_vtable, vtable const & empty_handler_vtable, - Allocator const a + Allocator const a, + publish_order const publish = {} ) { + // This path publishes through swap(), i.e. several plain stores, so + // it cannot honour a requested ordering: a concurrent prober could be + // handed an engaged vtable by one of the intermediate stores, with + // nothing ordering the target behind it (ThreadSanitizer confirms). + // The per-call opt-in therefore routes around it - callable::assign( + // f, memory_order ) clears and then takes the `direct` path, whose + // publication is a single store. Nothing should arrive here with an + // ordering request. + BOOST_ASSERT( !publish ); + boost::ignore_unused( publish ); // This most generic case needs to be reworked [currently does redundant // copying (through the vtable function pointers) and does not use all // the type information it could...]... using functor_manager = functor_manager, Allocator, buffer>; callable_base tmp( empty_handler_vtable, EmptyHandler() ); functor_manager::assign( std::forward( f ), tmp.functor_, a ); - tmp.store_vtable( &functor_vtable ); + tmp.store_vtable( functor_vtable ); this->swap( tmp, empty_handler_vtable ); } private: // Assignment from another functionoid helpers. - void assign_functionoid_direct( callable_base const & source, vtable const & /*empty_handler_vtable*/ ) noexcept( Traits::copyable >= support_level::nofail ) + void assign_functionoid_direct( callable_base const & source, vtable const & /*empty_handler_vtable*/, publish_order const publish = {} ) noexcept( Traits::copyable >= support_level::nofail ) { static_assert( Traits::copyable != support_level::na, "Callable not copyable" ); source.get_vtable().clone( source.functor_, this->functor_ ); - store_vtable( &source.get_vtable() ); + store_vtable( source.get_vtable(), publish ); } - void assign_functionoid_direct( callable_base && source, vtable const & empty_handler_vtable ) noexcept( ( Traits::moveable >= support_level::nofail ) || ( Traits::moveable == support_level::na && Traits::copyable >= support_level::nofail ) ) + void assign_functionoid_direct( callable_base && source, vtable const & empty_handler_vtable, publish_order const publish = {} ) noexcept( ( Traits::moveable >= support_level::nofail ) || ( Traits::moveable == support_level::na && Traits::copyable >= support_level::nofail ) ) { source.move_to( *this ); - this ->store_vtable( &source.get_vtable() ); - source.store_vtable( &empty_handler_vtable ); + this ->store_vtable( source.get_vtable(), publish ); + source.store_vtable( empty_handler_vtable ); } static constexpr bool compatible_vtable_function_entry( support_level const me, support_level const other ) noexcept @@ -1178,7 +1230,7 @@ class callable_base : public callable_tag } template - void assign_functionoid_direct( callable_base const & source, vtable const & /*empty_handler_vtable*/ ) noexcept( OtherTraits::copyable >= support_level::nofail ) + void assign_functionoid_direct( callable_base const & source, vtable const & /*empty_handler_vtable*/, publish_order const publish = {} ) noexcept( OtherTraits::copyable >= support_level::nofail ) { static_assert( compatible_vtables() ); static_assert( Traits::sbo_size >= OtherTraits::sbo_size ); @@ -1194,15 +1246,15 @@ class callable_base : public callable_tag auto & source_vtable{ source.get_vtable() } ; static_assert( sizeof( *p_vtable_ ) == sizeof( source_vtable ) ); - store_vtable( reinterpret_cast( &source_vtable ) ); + store_vtable( reinterpret_cast( source_vtable ), publish ); } template - void assign_functionoid_guarded( FunctionBaseRef && source, vtable const & empty_handler_vtable ) + void assign_functionoid_guarded( FunctionBaseRef && source, vtable const & empty_handler_vtable, publish_order const publish = {} ) { destroy(); cleaner guard( *this, empty_handler_vtable ); - assign_functionoid_direct( std::forward( source ), empty_handler_vtable ); + assign_functionoid_direct( std::forward( source ), empty_handler_vtable, publish ); guard.cancel(); } @@ -1232,12 +1284,7 @@ private: template friend class callable_base; class safe_mover_base; template class safe_mover; - // __restrict promises the pointer is not aliased - which is exactly what - // concurrent_reads says it is (and std::atomic_ref cannot bind to a - // restrict-qualified lvalue anyway), so the opt-in drops it. - // mutable: load_vtable() is const, and std::atomic_ref's const-T - // specialization is both unnecessary here and libc++-buggy. - mutable std::conditional_t p_vtable_; + vtable const * __restrict p_vtable_; mutable buffer functor_ ; }; // class callable_base @@ -1263,7 +1310,7 @@ class callable_base::safe_mover_base empty_function_to_move_to_{ empty_function_to_move_to }, empty_handler_vtable_ { empty_function_to_move_to.get_vtable() } { - BOOST_ASSERT( empty_function_to_move_to_.load_vtable( std::memory_order_relaxed ) == &empty_handler_vtable_ ); + BOOST_ASSERT( empty_function_to_move_to_.vtable_slot( std::memory_order_relaxed ) == &empty_handler_vtable_ ); move( function_to_guard, empty_function_to_move_to_, empty_handler_vtable_ ); } @@ -1274,7 +1321,7 @@ class callable_base::safe_mover_base { source.move_to( destination ); destination.store_vtable( source.load_vtable( std::memory_order_relaxed ) ); - source .store_vtable( &empty_handler_vtable ); + source .store_vtable( empty_handler_vtable ); } protected: diff --git a/include/psi/functionoid/functionoid.hpp b/include/psi/functionoid/functionoid.hpp index 207fa54..e80c543 100644 --- a/include/psi/functionoid/functionoid.hpp +++ b/include/psi/functionoid/functionoid.hpp @@ -186,11 +186,68 @@ class callable template void assign( F && f, Allocator const a ) { this->do_assign( std::forward( f ), a ); } + /// Assign and publish the new target with an explicit ordering - the + /// per-call counterpart of Traits::concurrent_reads, available whether or + /// not the Traits opted in (see publish_order in policies.hpp for what the + /// caller then owes). + /// memory_order_release is the meaningful choice: the target buffer is + /// written before the vtable pointer, so a release on that store is what + /// makes a concurrent empty( memory_order_acquire ) observer see a target + /// that is fully constructed. + /// \pre The callable is EMPTY. This overload ARMS a callable exactly + /// once; it is not a general ordered assignment, and the restriction is + /// the mechanism's, not an implementation shortcut: + /// * A release store publishes the writes that PRECEDE it. Re-arming an + /// already-armed callable would publish a target whose predecessor a + /// concurrent prober may have observed and be invoking right now, and + /// the reassignment destroys it under that reader. No memory ordering + /// closes that window - it needs deferred reclamation (refcounting, + /// RCU grace periods, hazard pointers), which a callable does not and + /// should not carry. + /// * So the only race-free transition is empty -> armed, once. Readers + /// poll empty( acquire ) and, on observing engagement, may use the + /// target for as long as it is never re-armed or cleared. + /// Implemented via the `direct` path, whose publication is EXACTLY ONE + /// store - the ordered one - and whose own precondition is this same + /// emptiness (it skips pre-destruction because there is nothing to + /// destroy). The ordinary route would not do: its general, may-throw case + /// publishes through swap(), i.e. several plain stores, one of which hands + /// a concurrent prober an engaged vtable with no ordering behind it + /// (ThreadSanitizer confirms the race). + template + void assign( F && f, std::memory_order const publish ) + { + BOOST_ASSERT_MSG( this->empty(), "ordered assign arms an EMPTY callable - re-arming races its readers" ); + this->do_assign( std::forward( f ), publish_order{ publish } ); + } + + template + void assign( F && f, Allocator const a, std::memory_order const publish ) + { + BOOST_ASSERT_MSG( this->empty(), "ordered assign arms an EMPTY callable - re-arming races its readers" ); + this->do_assign( std::forward( f ), a, publish_order{ publish } ); + } + void assign( std::nullptr_t ) noexcept { clear(); } /// Clear out a target (replace it with an empty handler), if there is one. void clear() { function_base:: template clear( empty_handler_vtable() ); } + /// There is deliberately no clear( std::memory_order ). A release store + /// orders the writes that precede it, and disengagement has none to order + /// - so an "ordered clear" would publish nothing while reading as though + /// it made disarming safe. It does not: a prober that observed engagement + /// beforehand may be invoking the target that clear() is destroying, which + /// is a reclamation problem, not an ordering one. + /// Same conclusion the Linux kernel reached for the identical shape: + /// rcu_assign_pointer() degrades to a plain WRITE_ONCE() when publishing + /// NULL, precisely because there is nothing to order (commit "rcu: No + /// ordering for rcu_assign_pointer() of NULL"). RCU then keeps removal on + /// an entirely separate mechanism - grace periods - rather than pretending + /// a store ordering can retire an object. + /// Disarm therefore requires the callable to be quiesced by other means, + /// at which point plain clear() is exactly right. + /// Determine if the function is empty (i.e. has an empty target). bool empty() const noexcept { return function_base::empty( &empty_handler_vtable() ); } @@ -204,7 +261,6 @@ class callable /// the assignment that published it. memory_order_relaxed answers the /// question alone and orders nothing else. bool empty( std::memory_order const order ) const noexcept - requires ( Traits::concurrent_reads ) { return function_base::empty( &empty_handler_vtable(), order ); } @@ -303,30 +359,31 @@ class callable // assigning but constructing) so it should probably be renamed to // pre_destroy or the whole thing solved in some smarter way... template - void do_assign( F && f, Allocator const a ) + requires ( !std::same_as ) // else a 2-arg do_assign( f, publish ) is ambiguous with the overload below + void do_assign( F && f, Allocator const a, publish_order const publish = {} ) { using tag = typename detail::get_function_tag::type; - dispatch_assign( std::forward( f ), a, tag{} ); + dispatch_assign( std::forward( f ), a, tag{}, publish ); } template - void do_assign( F && f ) + void do_assign( F && f, publish_order const publish = {} ) { using functor_type = std::remove_const_t>; using allocator = typename Traits:: template allocator; - do_assign( std::forward( f ), allocator{} ); + do_assign( std::forward( f ), allocator{}, publish ); } template - void dispatch_assign( F && f , Allocator const a, detail::function_obj_tag ) { do_assign( std::forward( f ), std::forward( f ), a ); } + void dispatch_assign( F && f , Allocator const a, detail::function_obj_tag , publish_order const publish = {} ) { do_assign( std::forward( f ), std::forward( f ), a, publish ); } // Explicit support for member function objects, so we invoke through // mem_fn() but retain the right target_type() values. template - void dispatch_assign( F const f, Allocator const a, detail::member_ptr_tag ) { do_assign( f , mem_fn( f ), a ); } + void dispatch_assign( F const f, Allocator const a, detail::member_ptr_tag , publish_order const publish = {} ) { do_assign( f , mem_fn( f ), a, publish ); } template - void dispatch_assign( F const f, Allocator const a, detail::function_obj_ref_tag ) { do_assign( f.get(), f , a ); } + void dispatch_assign( F const f, Allocator const a, detail::function_obj_ref_tag, publish_order const publish = {} ) { do_assign( f.get(), f , a, publish ); } template - void dispatch_assign( F f, Allocator const a, detail::function_ptr_tag ) + void dispatch_assign( F f, Allocator const a, detail::function_ptr_tag , publish_order const publish = {} ) { // Plain function pointers need special care because when assigned // using the syntax without the ampersand they wreck havoc with certain @@ -334,11 +391,11 @@ class callable // behaviour, e.g. not invoking the assigned target with GCC 4.0.1 or // causing access-violation crashes with MSVC (tested 8 and 10). using non_const_function_pointer_t = std::add_pointer_t>>; - do_assign( f, std::move( f ), a ); + do_assign( f, std::move( f ), a, publish ); } template - void do_assign( ActualFunctor const &, StoredFunctor && stored_functor, ActualFunctorAllocator const a ) + void do_assign( ActualFunctor const &, StoredFunctor && stored_functor, ActualFunctorAllocator const a, publish_order const publish ) { using NakedStoredFunctor = std::remove_const_t>; using StoredFunctorAllocator = typename std::allocator_traits::template rebind_alloc; @@ -347,7 +404,8 @@ class callable std::forward( stored_functor ), vtable_for_functor( stored_functor ), empty_handler_vtable(), - StoredFunctorAllocator( a ) + StoredFunctorAllocator( a ), + publish ); } }; // class callable diff --git a/include/psi/functionoid/policies.hpp b/include/psi/functionoid/policies.hpp index 4cc1a79..0636e4f 100644 --- a/include/psi/functionoid/policies.hpp +++ b/include/psi/functionoid/policies.hpp @@ -20,8 +20,10 @@ #include #include +#include #include #include +#include #include //------------------------------------------------------------------------------ namespace psi::functionoid @@ -69,6 +71,20 @@ template <> inline void assert_on_empty::handle_empty_invoke() noexcept { struct nop_on_empty { template static result_type handle_empty_invoke() noexcept { return {}; } }; template <> inline void nop_on_empty::handle_empty_invoke() noexcept {} +/// How an assignment publishes the new target: unset - the default - +/// defers to Traits::concurrent_reads; set, it is the per-call opt-in and the +/// publishing store is made atomically with the given ordering even for Traits +/// that did not opt in. +/// Use it when a callable type is overwhelmingly single-threaded but ONE +/// site needs the publish-once pattern: the trait is per-type and would tax +/// every other use (and, because it makes the invoke path atomic too, it costs +/// far more than the store - see concurrent_reads). This is the same trade +/// std::atomic_ref itself offers, and it carries the same caller obligation: +/// while an ordered access is in flight, every conflicting access to that +/// object must also be ordered. Prefer the trait when in doubt - it makes that +/// hold by construction rather than by discipline. +using publish_order = std::optional; + enum struct support_level : std::uint8_t { na = false, diff --git a/test/concurrent_reads_test.cpp b/test/concurrent_reads_test.cpp index 99cbcf4..729c42b 100644 --- a/test/concurrent_reads_test.cpp +++ b/test/concurrent_reads_test.cpp @@ -44,12 +44,27 @@ using plain_fn = pf::callable; using concurrent_fn = pf::callable; using trivial_fn = pf::callable; -// The overload is opt-in: absent unless the Traits ask for it. +// The ordered accessors are available to every Traits - asking for an ordering +// IS the per-call opt-in. What concurrent_reads adds is that the type's own +// internal accesses become ordered too, so the guarantee holds by construction +// rather than by caller discipline. template -concept has_ordered_empty = requires ( Callable const & c ) { c.empty( std::memory_order_acquire ); }; +concept has_ordered_empty = requires ( Callable const & c ) { c.empty( std::memory_order_acquire ); }; +template +concept has_ordered_assign = requires ( Callable & c ) { c.assign( +[]{ return 0; }, std::memory_order_release ); }; + +static_assert( has_ordered_empty ); +static_assert( has_ordered_empty ); +static_assert( has_ordered_assign ); +static_assert( has_ordered_assign ); + +// ...but there is deliberately NO ordered clear: a release store orders the +// writes that precede it, and disengagement has none. See clear()'s comment. +template +concept has_ordered_clear = requires ( Callable & c ) { c.clear( std::memory_order_release ); }; -static_assert( !has_ordered_empty ); -static_assert( has_ordered_empty ); +static_assert( !has_ordered_clear ); +static_assert( !has_ordered_clear ); // The whole point of publishing through the vtable pointer rather than an added // atomic member: no size cost, and no loss of triviality. @@ -151,3 +166,66 @@ TEST( ConcurrentReads, PublishOnceIsObservedWithItsTarget ) EXPECT_EQ( observed.load( std::memory_order_relaxed ), round * readers ); } } + +// --------------------------------------------------------------------------- +// Per-call opt-in: the same publish-once discipline on Traits that did NOT set +// concurrent_reads. The point is that a type used single-threaded everywhere +// else pays nothing - only this site is ordered. +// --------------------------------------------------------------------------- + +TEST( PerCallPublishOrder, OrderedEmptyAgreesWithPlainEmptyOnPlainTraits ) +{ + plain_fn f; + EXPECT_TRUE( f.empty( ) ); + EXPECT_TRUE( f.empty( std::memory_order_acquire ) ); + + f.assign( +[]{ return 5; }, std::memory_order_release ); + EXPECT_FALSE( f.empty( ) ); + EXPECT_FALSE( f.empty( std::memory_order_acquire ) ); + EXPECT_EQ ( f(), 5 ); + + f.clear(); + EXPECT_TRUE( f.empty( std::memory_order_acquire ) ); +} + +// The opt-in must not cost the type anything it did not already pay. +static_assert( sizeof( plain_fn ) == sizeof( concurrent_fn ) ); + +// Publish-once on plain Traits: identical to the concurrent_traits case above, +// but the ordering comes from the two annotated calls rather than the type. +TEST( PerCallPublishOrder, PublishOnceIsObservedWithItsTargetOnPlainTraits ) +{ + static constexpr auto readers{ 4 }; + + for ( auto round{ 0 }; round < 64; ++round ) + { + auto const p_function{ std::make_unique() }; + auto & function { *p_function }; + + std::atomic go{ false }; + std::atomic observed{ 0 }; + + std::vector pollers; + for ( auto reader{ 0 }; reader < readers; ++reader ) + { + pollers.emplace_back( [&] + { + go.wait( false, std::memory_order_acquire ); + while ( function.empty( std::memory_order_acquire ) ) { std::this_thread::yield(); } + observed.fetch_add( function(), std::memory_order_relaxed ); + } ); + } + + std::jthread writer{ [&] + { + go.store( true, std::memory_order_release ); + go.notify_all(); + function.assign( [ round ] { return round; }, std::memory_order_release ); + } }; + + pollers.clear(); // join + writer .join (); + + EXPECT_EQ( observed.load( std::memory_order_relaxed ), round * readers ); + } +}