diff --git a/include/exec/finally.hpp b/include/exec/finally.hpp index 7fc59e982..51a8d9fc5 100644 --- a/include/exec/finally.hpp +++ b/include/exec/finally.hpp @@ -75,7 +75,7 @@ namespace exec { void set_value() noexcept { STDEXEC_TRY { auto& __result = __op_->__result_.__get(); - __result.visit(__visitor{}, static_cast<_ResultType&&>(__result), __op_); + STDEXEC::__visit(__visitor{}, static_cast<_ResultType&&>(__result), __op_); } STDEXEC_CATCH_ALL { if constexpr (!__mapply_q<__nothrow_decay_copyable_t, _ResultType>::value) { @@ -185,7 +185,7 @@ namespace exec { .template emplace<__decayed_tuple<_Args...>>(static_cast<_Args&&>(__args)...); STDEXEC_ASSERT(__op_.index() == 0); auto __final = static_cast<_FinalSender&&>(__op_.template get<0>().__sndr_); - __final_op_t& __final_op = __op_.template emplace_from_at<1>( + __final_op_t& __final_op = __op_.template __emplace_from<1>( STDEXEC::connect, static_cast<_FinalSender&&>(__final), __final_receiver_t{this}); STDEXEC::start(__final_op); } diff --git a/include/exec/fork_join.hpp b/include/exec/fork_join.hpp index 3b036870c..197848ca9 100644 --- a/include/exec/fork_join.hpp +++ b/include/exec/fork_join.hpp @@ -95,7 +95,7 @@ namespace exec { using operation_state_concept = STDEXEC::operation_state_t; STDEXEC_ATTRIBUTE(host, device) void start() noexcept { - Variant::visit(_dematerialize_fn{}, *_results_, _rcvr_); + STDEXEC::__visit(_dematerialize_fn{}, *_results_, _rcvr_); } Rcvr _rcvr_; @@ -160,7 +160,7 @@ namespace exec { STDEXEC_ATTRIBUTE(host, device) ~_opstate_t() { // If this opstate was never started, we must explicitly destroy the _child_opstate_. - if (_cache_.is_valueless()) { + if (_cache_.__is_valueless()) { _child_opstate_.__destroy(); } } diff --git a/include/exec/sequence.hpp b/include/exec/sequence.hpp index 959d99367..1f8b79aea 100644 --- a/include/exec/sequence.hpp +++ b/include/exec/sequence.hpp @@ -114,7 +114,7 @@ namespace exec { // Below, it looks like we are using `sndrs` after it has been moved from. This is not the // case. `sndrs` is moved into a tuple type that has `__ignore` for the first element. The // result is that the first sender in `sndrs` is not moved from, but the rest are. - _ops.template emplace_from_at<0>( + _ops.template __emplace_from<0>( STDEXEC::connect, STDEXEC::__get<0>(static_cast(sndrs)), _rcvr_t<0>{this}); } @@ -127,7 +127,7 @@ namespace exec { STDEXEC::set_value(static_cast(_rcvr), static_cast(args)...); } else { auto& sndr = STDEXEC::__get(_sndrs); - auto& op = _ops.template emplace_from_at( + auto& op = _ops.template __emplace_from( STDEXEC::connect, std::move(sndr), _rcvr_t{this}); STDEXEC::start(op); } diff --git a/include/exec/sequence/ignore_all_values.hpp b/include/exec/sequence/ignore_all_values.hpp index c39b7725f..6ddafed34 100644 --- a/include/exec/sequence/ignore_all_values.hpp +++ b/include/exec/sequence/ignore_all_values.hpp @@ -59,22 +59,21 @@ namespace exec { [[gnu::noinline]] #endif void __visit_result(_Receiver __rcvr) noexcept { - int __is_emplaced = __emplaced_.load(__std::memory_order_acquire); - if (__is_emplaced == 0) { + if (__emplaced_.load(__std::memory_order_acquire) == 0) { STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr)); - return; + } else if constexpr (STDEXEC::__mapply::value != 0) { + STDEXEC_ASSERT(__result_.index() != __variant_npos); + STDEXEC::__visit( + [&](_Tuple&& __tuple) noexcept { + STDEXEC::__apply( + [&]<__completion_tag _Tag, class... _Args>( + _Tag __completion, _Args&&... __args) noexcept { + __completion(static_cast<_Receiver&&>(__rcvr), static_cast<_Args&&>(__args)...); + }, + static_cast<_Tuple&&>(__tuple)); + }, + static_cast<_ResultVariant&&>(__result_)); } - STDEXEC_ASSERT(__result_.index() != __variant_npos); - __result_.visit( - [&](_Tuple&& __tuple) noexcept { - STDEXEC::__apply( - [&]<__completion_tag _Tag, class... _Args>( - _Tag __completion, _Args&&... __args) noexcept { - __completion(static_cast<_Receiver&&>(__rcvr), static_cast<_Args&&>(__args)...); - }, - static_cast<_Tuple&&>(__tuple)); - }, - static_cast<_ResultVariant&&>(__result_)); } }; diff --git a/include/exec/sequence/iterate.hpp b/include/exec/sequence/iterate.hpp index ea7d652ca..f965c35e1 100644 --- a/include/exec/sequence/iterate.hpp +++ b/include/exec/sequence/iterate.hpp @@ -126,7 +126,7 @@ namespace exec { STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_)); } else { STDEXEC_TRY { - STDEXEC::start(__op_.emplace_from( + STDEXEC::start(__op_.__emplace_from( STDEXEC::connect, exec::set_next( __rcvr_, exec::sequence(STDEXEC::schedule(__scheduler_), __sender_t<_Range>{this})), diff --git a/include/exec/sequence/merge_each.hpp b/include/exec/sequence/merge_each.hpp index 9ac3ec443..6c25ffc2a 100644 --- a/include/exec/sequence/merge_each.hpp +++ b/include/exec/sequence/merge_each.hpp @@ -213,7 +213,7 @@ namespace exec { void start() & noexcept { // emit delayed error into the sequence - __op_->__error_storage_->visit( + STDEXEC::__visit( [this](auto&& __error) noexcept { STDEXEC::set_error( static_cast<_ErrorReceiver&&>(__receiver_), @@ -999,7 +999,7 @@ namespace exec { __nothrow_subscribable<_NestedSequence, __receiver_t> && STDEXEC::__nothrow_constructible_from<_NestedSeqOp, __nested_op_t>; STDEXEC_TRY { - auto& __nested_seq_op = __next_seq_op_->__nested_seq_op_.emplace_from( + auto& __nested_seq_op = __next_seq_op_->__nested_seq_op_.__emplace_from( subscribe, static_cast<_NestedSequence&&>(__sequence), __receiver_t{__next_seq_op_, __op_}); diff --git a/include/exec/when_any.hpp b/include/exec/when_any.hpp index 51f125231..bcda04ef2 100644 --- a/include/exec/when_any.hpp +++ b/include/exec/when_any.hpp @@ -139,8 +139,8 @@ namespace exec { STDEXEC::set_stopped(static_cast<_Receiver&&>(__rcvr_)); return; } - STDEXEC_ASSERT(!__result_.is_valueless()); - __result_.visit( + STDEXEC_ASSERT(!__result_.__is_valueless()); + STDEXEC::__visit( __when_any::__make_visitor_fn(__rcvr_), static_cast<_ResultVariant&&>(__result_)); } } diff --git a/include/nvexec/stream/when_all.cuh b/include/nvexec/stream/when_all.cuh index fa6b4b09a..0e643f3b2 100644 --- a/include/nvexec/stream/when_all.cuh +++ b/include/nvexec/stream/when_all.cuh @@ -321,7 +321,7 @@ namespace nvexec::_strm { } break; case _when_all::error: - errors_.visit( + STDEXEC::__visit( __mk_completion_fn(STDEXEC::set_error, rcvr_), static_cast(errors_)); break; diff --git a/include/stdexec/__detail/__continues_on.hpp b/include/stdexec/__detail/__continues_on.hpp index b07ac6fe5..5b9396d01 100644 --- a/include/stdexec/__detail/__continues_on.hpp +++ b/include/stdexec/__detail/__continues_on.hpp @@ -81,28 +81,32 @@ namespace STDEXEC { using __completions_t = __completions_impl_t<_Scheduler, __completion_signatures_of_t<_CvrefSender, _Env...>, _Env...>; - template - struct __state; - template STDEXEC_ATTRIBUTE(always_inline) - auto __make_visitor_fn(_State* __state) noexcept { + constexpr auto __make_visitor_fn(_State* __state) noexcept { return [__state](_Tup& __tupl) noexcept -> void { STDEXEC::__apply( - [&](auto __tag, _Args&... __args) noexcept -> void { - __tag(std::move(__state->__rcvr_), static_cast<_Args&&>(__args)...); + [&](_Tag, _Args&... __args) noexcept -> void { + _Tag()(static_cast<_State&&>(*__state).__rcvr_, static_cast<_Args&&>(__args)...); }, __tupl); }; } + template + struct __state_base : __immovable { + using __variant_t = __results_of<__child_of<_Sexpr>, env_of_t<_Receiver>>; + + _Receiver __rcvr_; + __variant_t __data_{}; + }; + // This receiver is to be completed on the execution context associated with the scheduler. When // the source sender completes, the completion information is saved off in the operation state // so that when this receiver completes, it can read the completion out of the operation state // and forward it to the output receiver after transitioning to the scheduler's context. - template + template struct __rcvr2 { - using _Scheduler = STDEXEC::__t<_SchedulerId>; using _Sexpr = STDEXEC::__t<_SexprId>; using _Receiver = STDEXEC::__t<_ReceiverId>; @@ -110,44 +114,42 @@ namespace STDEXEC { using receiver_concept = receiver_t; using __id = __rcvr2; - void set_value() noexcept { - __state_->__data_.visit(__trnsfr::__make_visitor_fn(__state_), __state_->__data_); + constexpr void set_value() noexcept { + STDEXEC::__visit(__trnsfr::__make_visitor_fn(__state_), __state_->__data_); } template - void set_error(_Error&& __err) noexcept { + constexpr void set_error(_Error&& __err) noexcept { STDEXEC::set_error( static_cast<_Receiver&&>(__state_->__rcvr_), static_cast<_Error&&>(__err)); } - void set_stopped() noexcept { + constexpr void set_stopped() noexcept { STDEXEC::set_stopped(static_cast<_Receiver&&>(__state_->__rcvr_)); } - auto get_env() const noexcept -> env_of_t<_Receiver> { + [[nodiscard]] + constexpr auto get_env() const noexcept -> env_of_t<_Receiver> { return STDEXEC::get_env(__state_->__rcvr_); } - __state<_Scheduler, _Sexpr, _Receiver>* __state_; + __state_base<_Sexpr, _Receiver>* __state_; }; }; - template - using __receiver2 = __t<__rcvr2<__id<_Scheduler>, __id<_Sexpr>, __id<_Receiver>>>; + template + using __receiver2 = __t<__rcvr2<__id<_Sexpr>, __id<_Receiver>>>; template - struct __state : __immovable { + struct __state : __state_base<_Sexpr, _Receiver> { using __variant_t = __results_of<__child_of<_Sexpr>, env_of_t<_Receiver>>; - using __receiver2_t = __receiver2<_Scheduler, _Sexpr, _Receiver>; + using __receiver2_t = __receiver2<_Sexpr, _Receiver>; - explicit __state(_Scheduler __sched, _Receiver&& __rcvr) - : __rcvr_(static_cast<_Receiver&&>(__rcvr)) - , __data_() + constexpr explicit __state(_Scheduler __sched, _Receiver&& __rcvr) + : __state::__state_base{{}, static_cast<_Receiver&&>(__rcvr), {}} , __state2_(connect(schedule(__sched), __receiver2_t{this})) { } - _Receiver __rcvr_; - __variant_t __data_; connect_result_t, __receiver2_t> __state2_; }; @@ -387,10 +389,10 @@ namespace STDEXEC { // Write the tag and the args into the operation state so that we can forward the completion // from within the scheduler's execution context. if constexpr (__nothrow_callable<__mktuple_t, _Tag, _Args...>) { - __state.__data_.emplace_from(__mktuple, __tag, static_cast<_Args&&>(__args)...); + __state.__data_.__emplace_from(__mktuple, __tag, static_cast<_Args&&>(__args)...); } else { STDEXEC_TRY { - __state.__data_.emplace_from(__mktuple, __tag, static_cast<_Args&&>(__args)...); + __state.__data_.__emplace_from(__mktuple, __tag, static_cast<_Args&&>(__args)...); } STDEXEC_CATCH_ALL { STDEXEC::set_error(static_cast<_State&&>(__state).__rcvr_, std::current_exception()); diff --git a/include/stdexec/__detail/__let.hpp b/include/stdexec/__detail/__let.hpp index d58729ed3..a45560c52 100644 --- a/include/stdexec/__detail/__let.hpp +++ b/include/stdexec/__detail/__let.hpp @@ -99,20 +99,21 @@ namespace STDEXEC { using __id = __rcvr_env; template - void set_value(_As&&... __as) noexcept { + constexpr void set_value(_As&&... __as) noexcept { STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), static_cast<_As&&>(__as)...); } template - void set_error(_Error&& __err) noexcept { + constexpr void set_error(_Error&& __err) noexcept { STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), static_cast<_Error&&>(__err)); } - void set_stopped() noexcept { + constexpr void set_stopped() noexcept { STDEXEC::set_stopped(static_cast<_Receiver&&>(__rcvr_)); } - auto get_env() const noexcept { + [[nodiscard]] + constexpr auto get_env() const noexcept { return __env::__join(__env_, STDEXEC::get_env(__rcvr_)); } @@ -276,97 +277,161 @@ namespace STDEXEC { >; }; + // A metafunction to check whether the predecessor's completion results are nothrow + // decay-copyable and whether connecting the secondary sender is nothrow. + template + struct __has_nothrow_completions_fn { + using __rcvr2_t = __receiver_archetype<_Env2>; + + template + using __f = __mbool< + __nothrow_decay_copyable<_Ts...> + && __nothrow_connectable<__call_result_t<_Fn, __decay_t<_Ts>&...>, __rcvr2_t> + >; + }; + + template + using __has_nothrow_completions_t = __gather_completions_t< + completion_signatures_of_t<_Child, _Env>, + _SetTag, + __has_nothrow_completions_fn<_Fn, __result_env_t<_SetTag, env_of_t<_Child>, _Env>>, + __qq<__mand_t> + >; + //! The core of the operation state for `let_*`. //! This gets bundled up into a larger operation state (`__detail::__op_state<...>`). - template - struct __state { - using __env2_t = - __let::__env2_t<_SetTag, env_of_t, env_of_t>; - using __second_rcvr_t = __receiver_with_env_t<_Receiver, __env2_t>; + template + struct __state_base : __immovable { + using __env2_t = _Env2; + using __second_rcvr_t = __receiver_with_env_t<_Receiver, _Env2>; + + template + constexpr explicit __state_base( + _SetTag, + const _Attrs& __attrs, + _Fun __fn, + _Receiver&& __rcvr) noexcept + : __rcvr_(static_cast<_Receiver&&>(__rcvr)) + , __fn_(static_cast<_Fun&&>(__fn)) + // TODO(ericniebler): this needs a fallback: + , __env2_(__let::__mk_env2<_SetTag>(__attrs, STDEXEC::get_env(__rcvr_))) { + } + + constexpr virtual void __start_next() = 0; template - constexpr void __impl(_Receiver& __rcvr, _Tag __tag, _Args&&... __args) noexcept { - if constexpr (std::is_same_v<_SetTag, _Tag>) { + constexpr void __impl(_Tag, _Args&&... __args) noexcept { + if constexpr (__same_as<_SetTag, _Tag>) { using __sender_t = __call_result_t<_Fun, __decay_t<_Args>&...>; - using __submit_t = __submit_result_t<__sender_t, __env2_t, _Receiver>; + using __submit_t = __submit_result_t<__sender_t, _Env2, _Receiver>; + constexpr bool __nothrow_store = (__nothrow_decay_copyable<_Args> && ...); constexpr bool __nothrow_invoke = __nothrow_callable<_Fun, __decay_t<_Args>&...>; - constexpr bool __nothrow_submit = noexcept( - __storage_ - .template emplace<__submit_t>(__declval<__sender_t>(), __declval<__second_rcvr_t>())); + constexpr bool __nothrow_submit = + __nothrow_constructible_from<__submit_t, __sender_t, __second_rcvr_t>; + STDEXEC_TRY { - auto& __tuple = __args_.emplace_from(__mktuple, static_cast<_Args&&>(__args)...); - auto&& __sender = ::STDEXEC::__apply(static_cast<_Fun&&>(__fn_), __tuple); - __storage_.template emplace<__monostate>(); - __second_rcvr_t __r{__rcvr, static_cast<__env2_t&&>(__env2_)}; - auto& __op = __storage_.template emplace<__submit_t>( - static_cast<__sender_t&&>(__sender), static_cast<__second_rcvr_t&&>(__r)); - __op.submit(static_cast<__sender_t&&>(__sender), static_cast<__second_rcvr_t&&>(__r)); + __args_.__emplace_from(__mktuple, static_cast<_Args&&>(__args)...); + __start_next(); } STDEXEC_CATCH_ALL { if constexpr (!(__nothrow_store && __nothrow_invoke && __nothrow_submit)) { - ::STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr), std::current_exception()); + STDEXEC::set_error(static_cast<_Receiver&&>(this->__rcvr_), std::current_exception()); } } } else { - __tag(static_cast<_Receiver&&>(__rcvr), static_cast<_Args&&>(__args)...); + _Tag()(static_cast<_Receiver&&>(this->__rcvr_), static_cast<_Args&&>(__args)...); } } - struct __first_rcvr_t { - using receiver_concept = ::STDEXEC::receiver_t; + _Receiver __rcvr_; + STDEXEC_IMMOVABLE_NO_UNIQUE_ADDRESS + _Fun __fn_; + STDEXEC_IMMOVABLE_NO_UNIQUE_ADDRESS + _Env2 __env2_; + //! Variant to hold the child sender's results before passing them to the function: + __variant_for<_Tuples...> __args_{}; + }; - template - constexpr void set_value(_Args&&... __args) noexcept { - __state_.__impl(__state_.__rcvr_, ::STDEXEC::set_value, static_cast<_Args&&>(__args)...); - } - template - constexpr void set_error(_Args&&... __args) noexcept { - __state_.__impl(__state_.__rcvr_, ::STDEXEC::set_error, static_cast<_Args&&>(__args)...); - } - template - constexpr void set_stopped(_Args&&... __args) noexcept { - __state_ - .__impl(__state_.__rcvr_, ::STDEXEC::set_stopped, static_cast<_Args&&>(__args)...); - } - // TODO(ericniebler): make this constexpr - //constexpr - auto get_env() const noexcept -> env_of_t<_Receiver> { - return ::STDEXEC::get_env(__state_.__rcvr_); - } + template + struct __first_rcvr { + using receiver_concept = STDEXEC::receiver_t; + using __t = __first_rcvr; + using __id = __first_rcvr; - __state& __state_; - }; + template + constexpr void set_value(_Args&&... __args) noexcept { + __state_->__impl(STDEXEC::set_value, static_cast<_Args&&>(__args)...); + } - using __result_variant = __variant_for<__monostate, _Tuples...>; - using __op_state_variant = __variant_for< - __monostate, - ::STDEXEC::connect_result_t<_Sender, __first_rcvr_t>, + template + constexpr void set_error(_Args&&... __args) noexcept { + __state_->__impl(STDEXEC::set_error, static_cast<_Args&&>(__args)...); + } + + template + constexpr void set_stopped(_Args&&... __args) noexcept { + __state_->__impl(STDEXEC::set_stopped, static_cast<_Args&&>(__args)...); + } + + [[nodiscard]] + constexpr auto get_env() const noexcept -> env_of_t<_Receiver> { + return STDEXEC::get_env(__state_->__rcvr_); + } + + __state_base<_SetTag, _Fun, _Receiver, _Env2, _Tuples...>* __state_; + }; + + //! The core of the operation state for `let_*`. + //! This gets bundled up into a larger operation state (`__detail::__op_state<...>`). + template + struct __state + : __state_base< + _SetTag, + _Fun, + _Receiver, + __let::__env2_t<_SetTag, env_of_t<_Child>, env_of_t<_Receiver>>, + _Tuples... + > { + using __env2_t = __state::__state_base::__env2_t; + using __first_rcvr_t = __first_rcvr<_SetTag, _Fun, _Receiver, __env2_t, _Tuples...>; + using __second_rcvr_t = __state::__state_base::__second_rcvr_t; + + using __op_state_variant_t = __variant_for< + connect_result_t<_Child, __first_rcvr_t>, __mapply<__submit_datum_for<_Receiver, _Fun, _SetTag, __env2_t>, _Tuples>... >; - constexpr explicit __state(_Sender&& __sender, _Fun __fn, _Receiver&& __rcvr) noexcept( - __nothrow_connectable<_Sender, __first_rcvr_t> - && std::is_nothrow_move_constructible_v<_Fun>) - : __rcvr_(static_cast<_Receiver&&>(__rcvr)) - , __fn_(static_cast<_Fun&&>(__fn)) - , __env2_( - // TODO(ericniebler): this needs a fallback - __let::__mk_env2<_SetTag>(::STDEXEC::get_env(__sender), ::STDEXEC::get_env(__rcvr_))) { - __storage_.emplace_from( - ::STDEXEC::connect, static_cast<_Sender&&>(__sender), __first_rcvr_t{*this}); + constexpr explicit __state(_Child&& __child, _Fun __fn, _Receiver&& __rcvr) noexcept( + __nothrow_connectable<_Child, __first_rcvr_t> && std::is_nothrow_move_constructible_v<_Fun>) + : __state::__state_base( + _SetTag(), + STDEXEC::get_env(__child), + static_cast<_Fun&&>(__fn), + static_cast<_Receiver&&>(__rcvr)) { + __storage_ + .__emplace_from(STDEXEC::connect, static_cast<_Child&&>(__child), __first_rcvr_t{this}); + } + + constexpr void __start_next() final { + STDEXEC_ASSERT(__storage_.index() == 0); + if constexpr (sizeof...(_Tuples) != 0) { + STDEXEC::__visit( + [this](auto& __tupl) { + using __sender_t = __apply_result_t<_Fun, decltype(__tupl)>; + auto&& __sndr = STDEXEC::__apply(static_cast<_Fun&&>(this->__fn_), __tupl); + using __submit_t = __submit_result_t<__sender_t, __env2_t, _Receiver>; + __second_rcvr_t __r{this->__rcvr_, static_cast<__env2_t&&>(this->__env2_)}; + auto& __op = __storage_.template emplace<__submit_t>( + static_cast<__sender_t&&>(__sndr), static_cast<__second_rcvr_t&&>(__r)); + __op.submit(static_cast<__sender_t&&>(__sndr), static_cast<__second_rcvr_t&&>(__r)); + }, + this->__args_); + } } - STDEXEC_IMMOVABLE_NO_UNIQUE_ADDRESS - _Receiver __rcvr_; - STDEXEC_IMMOVABLE_NO_UNIQUE_ADDRESS - _Fun __fn_; - STDEXEC_IMMOVABLE_NO_UNIQUE_ADDRESS - __env2_t __env2_; - //! Variant to hold the results passed from upstream before passing them to the function: - __result_variant __args_{}; //! Variant type for holding the operation state of the currently in flight operation - __op_state_variant __storage_{}; + __op_state_variant_t __storage_{}; }; // The set_value completions of: @@ -410,28 +475,6 @@ namespace STDEXEC { // * the error completions of the predecessor sender // * the error completions of the secondary senders - // A metafunction to check whether the predecessor's completion results are nothrow - // decay-copyable and whether connecting the secondary sender is nothrow. - template - struct __has_nothrow_completions_fn { - using __env2_t = __let::__result_env_t<_SetTag, env_of_t<_Sndr>, _Env>; - using __rcvr2_t = __receiver_archetype<__env2_t>; - - template - using __f = __mbool< - __nothrow_decay_copyable<_Ts...> - && __nothrow_connectable<__call_result_t<_Fn, __decay_t<_Ts>&...>, __rcvr2_t> - >; - }; - - template - using __has_nothrow_completions = __gather_completions_t< - completion_signatures_of_t<_Sndr, _Env>, - _SetTag, - __has_nothrow_completions_fn<_SetTag, _Sndr, _Fn, _Env>, - __qq<__mand_t> - >; - template struct __result_completion_behavior_fn { template @@ -511,7 +554,7 @@ namespace STDEXEC { } template <__one_of _Tag, class... _Env> - requires(__has_nothrow_completions<__set_tag_t, _Sndr, _Fn, _Env>::value && ...) + requires(__has_nothrow_completions_t<__set_tag_t, _Sndr, _Fn, _Env>::value && ...) [[nodiscard]] constexpr auto query(get_completion_domain_t<_Tag>, const _Env&...) const noexcept -> __ensure_valid_domain_t<__common_domain_t< @@ -522,7 +565,7 @@ namespace STDEXEC { } template - requires(!__has_nothrow_completions<__set_tag_t, _Sndr, _Fn, _Env>::value) + requires(!__has_nothrow_completions_t<__set_tag_t, _Sndr, _Fn, _Env>::value) [[nodiscard]] constexpr auto query(get_completion_domain_t, const _Env&) const noexcept -> __ensure_valid_domain_t<__common_domain_t< @@ -581,19 +624,19 @@ namespace STDEXEC { template struct __let_t { template - auto operator()(_Sender&& __sndr, _Fun __fn) const -> __well_formed_sender auto { + constexpr auto operator()(_Sender&& __sndr, _Fun __fn) const -> __well_formed_sender auto { return __make_sexpr<__let_t<_SetTag>>( static_cast<_Fun&&>(__fn), static_cast<_Sender&&>(__sndr)); } template STDEXEC_ATTRIBUTE(always_inline) - auto operator()(_Fun __fn) const { + constexpr auto operator()(_Fun __fn) const { return __closure(*this, static_cast<_Fun&&>(__fn)); } template - static auto transform_sender(set_value_t, _Sender&& __sndr, __ignore) { + static constexpr auto transform_sender(set_value_t, _Sender&& __sndr, __ignore) { if constexpr (__decay_copyable<_Sender>) { auto& [__tag, __fn, __child] = __sndr; return __make_sexpr<__let_tag<_SetTag>>(__data{ @@ -609,7 +652,7 @@ namespace STDEXEC { static constexpr auto get_attrs = []( const __data<_Child, _Fun>& __data) noexcept -> decltype(auto) { - // BUGBUG: TODO(ericniebler): this needs a proper implementation + // TODO(ericniebler): this needs a proper implementation return __fwd_env(STDEXEC::get_env(__data.__sndr)); }; @@ -636,28 +679,31 @@ namespace STDEXEC { static constexpr auto get_state = [](_Sender&& __sndr, _Receiver&& __rcvr) - requires sender_in<__child_of_t<_Sender>, env_of_t<_Receiver>> + requires sender_in<__child_of_t<_Sender>, __fwd_env_t>> + // TODO(ericniebler): make this conditionally noexcept { static_assert(sender_expr_for<_Sender, __let_tag<_SetTag>>); - using _Child = __child_of_t<_Sender>; - using _Fun = __decay_t<__fn_of_t<_Sender>>; - using __mk_let_state = __mbind_front_q<__state, _SetTag, _Child, _Fun, _Receiver>; - using __let_state_t = __gather_completions_of_t< + using __child_t = __child_of_t<_Sender>; + using __fn_t = __decay_t<__fn_of_t<_Sender>>; + using __mk_state = __mbind_front_q<__state, _SetTag, __child_t, __fn_t, _Receiver>; + using __state_t = __gather_completions_of_t< _SetTag, - _Child, - env_of_t<_Receiver>, + __child_t, + __fwd_env_t>, __q<__decayed_tuple>, - __mk_let_state + __mk_state >; - auto&& [__tag, __data] = static_cast<_Sender&&>(__sndr); - return __let_state_t( - __forward_like<_Sender>(__data).__sndr, - __forward_like<_Sender>(__data).__fn, + auto& [__tag, __data] = __sndr; + auto& [__child, __fn] = __data; + return __state_t( + STDEXEC::__forward_like<_Sender>(__child), + STDEXEC::__forward_like<_Sender>(__fn), static_cast<_Receiver&&>(__rcvr)); }; static constexpr auto start = [](_State& __state) noexcept { - ::STDEXEC::start(__state.__storage_.template get<1>()); + STDEXEC_ASSERT(__state.__storage_.index() == 0); + STDEXEC::start(__state.__storage_.template get<0>()); }; }; } // namespace __let diff --git a/include/stdexec/__detail/__on.hpp b/include/stdexec/__detail/__on.hpp index 87cfe53e7..59d4ef861 100644 --- a/include/stdexec/__detail/__on.hpp +++ b/include/stdexec/__detail/__on.hpp @@ -119,7 +119,7 @@ namespace STDEXEC { auto __pred = __reschedule(STDEXEC::__forward_like<_Sender>(__child), __old, __sched); return __reschedule( - __forward_like<_Sender>(__clsur)(std::move(__pred)), + STDEXEC::__forward_like<_Sender>(__clsur)(std::move(__pred)), std::move(__sched), std::move(__old)); } diff --git a/include/stdexec/__detail/__scope.hpp b/include/stdexec/__detail/__scope.hpp index ec62b73d5..014b90bd3 100644 --- a/include/stdexec/__detail/__scope.hpp +++ b/include/stdexec/__detail/__scope.hpp @@ -30,12 +30,12 @@ namespace STDEXEC { STDEXEC_ATTRIBUTE(no_unique_address) __immovable __hidden_ { }; bool __dismissed_{false}; - ~__scope_guard() { + constexpr ~__scope_guard() { if (!__dismissed_) static_cast<_Fn&&>(__fn_)(); } - void __dismiss() noexcept { + constexpr void __dismiss() noexcept { __dismissed_ = true; } }; @@ -48,11 +48,11 @@ namespace STDEXEC { bool __dismissed_{false}; - void __dismiss() noexcept { + constexpr void __dismiss() noexcept { __dismissed_ = true; } - ~__scope_guard() { + constexpr ~__scope_guard() { if (!__dismissed_) static_cast<_Fn&&>(__fn_)(static_cast<_T0&&>(__t0_)); } @@ -67,11 +67,11 @@ namespace STDEXEC { bool __dismissed_{false}; - void __dismiss() noexcept { + constexpr void __dismiss() noexcept { __dismissed_ = true; } - ~__scope_guard() { + constexpr ~__scope_guard() { if (!__dismissed_) static_cast<_Fn&&>(__fn_)(static_cast<_T0&&>(__t0_), static_cast<_T1&&>(__t1_)); } @@ -87,11 +87,11 @@ namespace STDEXEC { bool __dismissed_{false}; - void __dismiss() noexcept { + constexpr void __dismiss() noexcept { __dismissed_ = true; } - ~__scope_guard() { + constexpr ~__scope_guard() { if (!__dismissed_) static_cast<_Fn&&>( __fn_)(static_cast<_T0&&>(__t0_), static_cast<_T1&&>(__t1_), static_cast<_T2&&>(__t2_)); @@ -99,5 +99,5 @@ namespace STDEXEC { }; template - __scope_guard(_Fn, _Ts...) -> __scope_guard<_Fn, _Ts...>; + STDEXEC_HOST_DEVICE_DEDUCTION_GUIDE __scope_guard(_Fn, _Ts...) -> __scope_guard<_Fn, _Ts...>; } // namespace STDEXEC diff --git a/include/stdexec/__detail/__shared.hpp b/include/stdexec/__detail/__shared.hpp index 3a97b4a2f..05d59793b 100644 --- a/include/stdexec/__detail/__shared.hpp +++ b/include/stdexec/__detail/__shared.hpp @@ -152,7 +152,7 @@ namespace STDEXEC::__shared { __self->__on_stop_.reset(); - __variant_t::visit( + STDEXEC::__visit( __notify_visitor(), static_cast<__cv_variant_t&&>(__self->__sh_state_->__results_), __self->__rcvr_); diff --git a/include/stdexec/__detail/__utility.hpp b/include/stdexec/__detail/__utility.hpp index 3a45fd6ea..f45f76b35 100644 --- a/include/stdexec/__detail/__utility.hpp +++ b/include/stdexec/__detail/__utility.hpp @@ -17,10 +17,8 @@ #include "__concepts.hpp" #include "__config.hpp" -#include "__type_traits.hpp" #include -#include namespace STDEXEC { constexpr std::size_t __npos = ~0UL; @@ -131,19 +129,11 @@ namespace STDEXEC { return __pos_of(__same, __same + sizeof...(_Ts)); } - namespace __detail { - template - struct __forward_like_fn { - template - STDEXEC_ATTRIBUTE(always_inline) - constexpr auto operator()(_Uy&& __uy) const noexcept -> auto&& { - return static_cast<_Cpcvref::template __f>>(__uy); - } - }; - } // namespace __detail - - template - inline constexpr __detail::__forward_like_fn<__copy_cvref_fn<_Ty&&>> __forward_like{}; + template + STDEXEC_ATTRIBUTE(nodiscard, always_inline) + constexpr auto __forward_like(_Uy&& __uy) noexcept -> auto&& { + return static_cast<__copy_cvref_t<_Ty&&, STDEXEC_REMOVE_REFERENCE(_Uy)>>(__uy); + } STDEXEC_PRAGMA_PUSH() STDEXEC_PRAGMA_IGNORE_GNU("-Wold-style-cast") diff --git a/include/stdexec/__detail/__variant.hpp b/include/stdexec/__detail/__variant.hpp index 28eec6b99..82705187e 100644 --- a/include/stdexec/__detail/__variant.hpp +++ b/include/stdexec/__detail/__variant.hpp @@ -21,7 +21,9 @@ #include "__utility.hpp" #include +#include #include +#include #include /********************************************************************************/ @@ -46,14 +48,8 @@ namespace STDEXEC { struct __monostate { }; namespace __var { - template - STDEXEC_ATTRIBUTE(host, device, always_inline) - void __destroy_at(_Ty *ptr) noexcept { - ptr->~_Ty(); - } - STDEXEC_ATTRIBUTE(host, device) - inline auto __mk_index_guard(std::size_t &__index, std::size_t __new) noexcept { + constexpr auto __mk_index_guard(std::size_t &__index, std::size_t __new) noexcept { __index = __new; return __scope_guard{[&__index]() noexcept { __index = __variant_npos; }}; } @@ -70,11 +66,13 @@ namespace STDEXEC { STDEXEC_ASSERT(false); } - STDEXEC_ATTRIBUTE(host, device) static constexpr auto index() noexcept -> std::size_t { + STDEXEC_ATTRIBUTE(host, device) + static constexpr auto index() noexcept -> std::size_t { return __variant_npos; } - STDEXEC_ATTRIBUTE(host, device) static constexpr auto is_valueless() noexcept -> bool { + STDEXEC_ATTRIBUTE(host, device) + static constexpr auto __is_valueless() noexcept -> bool { return true; } }; @@ -86,10 +84,11 @@ namespace STDEXEC { std::size_t __index_{__variant_npos}; alignas(_Ts...) unsigned char __storage_[__max_size]; - STDEXEC_ATTRIBUTE(host, device) void __destroy() noexcept { + STDEXEC_ATTRIBUTE(host, device) + constexpr void __destroy() noexcept { auto __index = std::exchange(__index_, __variant_npos); if (__variant_npos != __index) { - ((_Is == __index ? __var::__destroy_at(static_cast<_Ts *>(__get_ptr())) : void(0)), ...); + ((_Is == __index ? std::destroy_at(static_cast<_Ts *>(__get_ptr())) : void(0)), ...); } } @@ -100,24 +99,33 @@ namespace STDEXEC { // immovable: __variant(__variant &&) = delete; - STDEXEC_ATTRIBUTE(host, device) __variant() noexcept = default; + STDEXEC_ATTRIBUTE(host, device) + __variant() noexcept = default; - STDEXEC_ATTRIBUTE(host, device) ~__variant() { + STDEXEC_ATTRIBUTE(host, device) + constexpr ~__variant() { __destroy(); } [[nodiscard]] - STDEXEC_ATTRIBUTE(host, device, always_inline) auto __get_ptr() noexcept -> void * { + STDEXEC_ATTRIBUTE(host, device, always_inline) constexpr auto __get_ptr() noexcept -> void * { return __storage_; } [[nodiscard]] - STDEXEC_ATTRIBUTE(host, device, always_inline) auto index() const noexcept -> std::size_t { + STDEXEC_ATTRIBUTE(host, device, always_inline) constexpr auto __get_ptr() const noexcept -> const void * { + return __storage_; + } + + [[nodiscard]] + STDEXEC_ATTRIBUTE(host, device, always_inline) constexpr auto index() const noexcept + -> std::size_t { return __index_; } [[nodiscard]] - STDEXEC_ATTRIBUTE(host, device, always_inline) auto is_valueless() const noexcept -> bool { + STDEXEC_ATTRIBUTE(host, device, always_inline) constexpr auto __is_valueless() const noexcept + -> bool { return __index_ == __variant_npos; } @@ -136,60 +144,73 @@ namespace STDEXEC { // must be aware of the danger. template STDEXEC_ATTRIBUTE(host, device) - auto emplace(_As &&...__as) noexcept(__nothrow_constructible_from<_Ty, _As...>) -> _Ty & { + constexpr auto emplace(_As &&...__as) noexcept(__nothrow_constructible_from<_Ty, _As...>) + -> _Ty & { constexpr std::size_t __new_index = STDEXEC::__index_of<_Ty, _Ts...>(); static_assert(__new_index != __variant_npos, "Type not in variant"); __destroy(); auto __sg = __mk_index_guard(__index_, __new_index); - auto *__p = ::new (__storage_) _Ty{static_cast<_As &&>(__as)...}; + auto *__p = + std::construct_at(static_cast<_Ty *>(__get_ptr()), static_cast<_As &&>(__as)...); __sg.__dismiss(); return *std::launder(__p); } template STDEXEC_ATTRIBUTE(host, device) - auto emplace(_As &&...__as) noexcept(__nothrow_constructible_from<__at<_Ny>, _As...>) - -> __at<_Ny> & { + constexpr auto emplace(_As &&...__as) + noexcept(__nothrow_constructible_from<__at<_Ny>, _As...>) -> __at<_Ny> & { static_assert(_Ny < sizeof...(_Ts), "variant index is too large"); __destroy(); auto __sg = __mk_index_guard(__index_, _Ny); - auto *__p = ::new (__storage_) __at<_Ny>{static_cast<_As &&>(__as)...}; + auto *__p = + std::construct_at(static_cast<__at<_Ny> *>(__get_ptr()), static_cast<_As &&>(__as)...); __sg.__dismiss(); return *std::launder(__p); } template STDEXEC_ATTRIBUTE(host, device) - auto emplace_from_at(_Fn &&__fn, _As &&...__as) noexcept(__nothrow_callable<_Fn, _As...>) + constexpr auto __emplace_from(_Fn &&__fn, _As &&...__as) noexcept(__nothrow_callable<_Fn, _As...>) -> __at<_Ny> & { static_assert( __same_as<__call_result_t<_Fn, _As...>, __at<_Ny>>, "callable does not return the correct type"); + constexpr bool __is_nothrow = __nothrow_callable<_Fn, _As...>; __destroy(); auto __sg = __mk_index_guard(__index_, _Ny); - auto *__p = ::new (__storage_) - __at<_Ny>(static_cast<_Fn &&>(__fn)(static_cast<_As &&>(__as)...)); - __sg.__dismiss(); - return *std::launder(__p); + if (std::is_constant_evaluated()) { + auto *__p = std::construct_at<__at<_Ny>>( + static_cast<__at<_Ny> *>(__get_ptr()), + STDEXEC::__emplace_from([&]() noexcept(__is_nothrow) -> decltype(auto) { + return static_cast<_Fn &&>(__fn)(static_cast<_As &&>(__as)...); + })); + __sg.__dismiss(); + return *std::launder(__p); + } else { + auto *__p = ::new (__get_ptr()) + __at<_Ny>(static_cast<_Fn &&>(__fn)(static_cast<_As &&>(__as)...)); + __sg.__dismiss(); + return *std::launder(__p); + } } template STDEXEC_ATTRIBUTE(host, device, always_inline) - auto emplace_from(_Fn &&__fn, _As &&...__as) noexcept(__nothrow_callable<_Fn, _As...>) - -> __call_result_t<_Fn, _As...> & { + constexpr auto __emplace_from(_Fn &&__fn, _As &&...__as) + noexcept(__nothrow_callable<_Fn, _As...>) -> __call_result_t<_Fn, _As...> & { using __result_t = __call_result_t<_Fn, _As...>; constexpr std::size_t __new_index = STDEXEC::__index_of<__result_t, _Ts...>(); static_assert(__new_index != __variant_npos, "Type not in variant"); - return emplace_from_at<__new_index>( - static_cast<_Fn &&>(__fn), static_cast<_As &&>(__as)...); + return __emplace_from<__new_index>(static_cast<_Fn &&>(__fn), static_cast<_As &&>(__as)...); } template STDEXEC_ATTRIBUTE(host, device) - static void visit(_Fn &&__fn, _Self &&__self, _As &&...__as) + static constexpr void __visit(_Fn &&__fn, _Self &&__self, _As &&...__as) noexcept((__nothrow_callable<_Fn, _As..., __copy_cvref_t<_Self, _Ts>> && ...)) { STDEXEC_ASSERT(__self.__index_ != __variant_npos); auto __index = __self.__index_; // make it local so we don't access it after it's deleted. @@ -202,23 +223,23 @@ namespace STDEXEC { template STDEXEC_ATTRIBUTE(nodiscard, host, device, always_inline) - auto get() && noexcept -> decltype(auto) { + constexpr auto get() && noexcept -> decltype(auto) { STDEXEC_ASSERT(_Ny == __index_); - return static_cast<__at<_Ny> &&>(*reinterpret_cast<__at<_Ny> *>(__storage_)); + return static_cast<__at<_Ny> &&>(*static_cast<__at<_Ny> *>(__get_ptr())); } template STDEXEC_ATTRIBUTE(nodiscard, host, device, always_inline) - auto get() & noexcept -> decltype(auto) { + constexpr auto get() & noexcept -> decltype(auto) { STDEXEC_ASSERT(_Ny == __index_); - return *reinterpret_cast<__at<_Ny> *>(__storage_); + return *static_cast<__at<_Ny> *>(__get_ptr()); } template STDEXEC_ATTRIBUTE(nodiscard, host, device, always_inline) - auto get() const & noexcept -> decltype(auto) { + constexpr auto get() const & noexcept -> decltype(auto) { STDEXEC_ASSERT(_Ny == __index_); - return *reinterpret_cast *>(__storage_); + return *static_cast *>(__get_ptr()); } }; } // namespace __var @@ -229,8 +250,8 @@ namespace STDEXEC { // clang-format off template STDEXEC_ATTRIBUTE(host, device, always_inline) - auto operator()(_Fn &&__fn, _Variant &&__var, _As &&...__as) const STDEXEC_AUTO_RETURN( - __var.visit( + constexpr auto operator()(_Fn &&__fn, _Variant &&__var, _As &&...__as) const STDEXEC_AUTO_RETURN( + __var.__visit( static_cast<_Fn &&>(__fn), static_cast<_Variant &&>(__var), static_cast<_As &&>(__as)...) diff --git a/include/stdexec/__detail/__when_all.hpp b/include/stdexec/__detail/__when_all.hpp index dbd4117d6..6aacc0ce6 100644 --- a/include/stdexec/__detail/__when_all.hpp +++ b/include/stdexec/__detail/__when_all.hpp @@ -231,7 +231,7 @@ namespace STDEXEC { case __error: if constexpr (!__same_as<_ErrorsVariant, __variant_for<>>) { // One or more child operations completed with an error: - __errors_.visit( + STDEXEC::__visit( __mk_completion_fn(set_error, __rcvr_), static_cast<_ErrorsVariant&&>(__errors_)); } break;