diff options
| author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:14:58 +0000 |
|---|---|---|
| committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:14:58 +0000 |
| commit | 5b3f540b862b99efb1cef5415c655e1495f2f522 (patch) | |
| tree | 3f713b8811219fff679f3a70c58574cc228f0412 | |
| parent | d678ff8adc8991646ff27b2d9fcc09d732a1bf09 (diff) | |
| parent | 8079887857e71037d3884d47cfecf4bc1584613d (diff) | |
| download | tokio-android14-mainline-sdkext-release.tar.gz | |
Snap for 10453563 from 8079887857e71037d3884d47cfecf4bc1584613d to mainline-sdkext-releaseaml_sdk_341510000aml_sdk_341410000aml_sdk_341110080aml_sdk_341110000aml_sdk_341010000aml_sdk_340912010android14-mainline-sdkext-release
Change-Id: I79c283f0e8517972a4a7dcabe8fbc5f21463d62c
329 files changed, 22622 insertions, 7461 deletions
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json index 92e2ebe..fb46d1e 100644 --- a/.cargo_vcs_info.json +++ b/.cargo_vcs_info.json @@ -1,5 +1,6 @@ { "git": { - "sha1": "623c09c52c2c38a8d75e94c166593547e8477707" - } -} + "sha1": "88b1eb54fb66461b9f3524f4b5316241a019279a" + }, + "path_in_vcs": "tokio" +}
\ No newline at end of file @@ -18,12 +18,11 @@ license { ], } -rust_library { - name: "libtokio", +rust_defaults { + name: "tokio_defaults", host_supported: true, - crate_name: "tokio", cargo_env_compat: true, - cargo_pkg_version: "1.14.0", + cargo_pkg_version: "1.25.0", srcs: ["src/lib.rs"], edition: "2018", features: [ @@ -38,12 +37,12 @@ rust_library { "num_cpus", "rt", "rt-multi-thread", + "socket2", "sync", "time", "tokio-macros", - "winapi", + "windows-sys", ], - cfgs: ["tokio_track_caller"], rustlibs: [ "libbytes", "liblibc", @@ -51,20 +50,37 @@ rust_library { "libmio", "libnum_cpus", "libpin_project_lite", + "libsocket2", ], proc_macros: ["libtokio_macros"], apex_available: [ "//apex_available:platform", - "com.android.bluetooth", + "com.android.btservices", "com.android.resolv", "com.android.uwb", ], + product_available: true, vendor_available: true, min_sdk_version: "29", } +rust_library { + name: "libtokio", + crate_name: "tokio", + defaults: ["tokio_defaults"], +} + +rust_library { + name: "libtokio_for_test", + crate_name: "tokio", + defaults: ["tokio_defaults"], + features: [ + "test-util", + ], +} + rust_defaults { - name: "tokio_defaults_tokio", + name: "tokio_defaults_tests", crate_name: "tokio", cargo_env_compat: true, test_suites: ["general-tests"], @@ -108,7 +124,7 @@ rust_defaults { rust_test { name: "tokio_test_tests__require_full", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/_require_full.rs"], test_options: { @@ -118,7 +134,7 @@ rust_test { rust_test { name: "tokio_test_tests_buffered", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/buffered.rs"], test_options: { @@ -128,7 +144,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_async_fd", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_async_fd.rs"], test_options: { @@ -138,7 +154,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_async_read", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_async_read.rs"], test_options: { @@ -148,7 +164,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_chain", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_chain.rs"], test_options: { @@ -158,7 +174,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_copy", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_copy.rs"], test_options: { @@ -168,7 +184,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_copy_bidirectional", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_copy_bidirectional.rs"], test_options: { @@ -178,7 +194,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_driver", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_driver.rs"], test_options: { @@ -188,7 +204,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_driver_drop", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_driver_drop.rs"], test_options: { @@ -198,7 +214,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_lines", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_lines.rs"], test_options: { @@ -208,7 +224,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_mem_stream", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_mem_stream.rs"], test_options: { @@ -218,7 +234,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read.rs"], test_options: { @@ -228,7 +244,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_buf", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_buf.rs"], test_options: { @@ -238,7 +254,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_exact", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_exact.rs"], test_options: { @@ -248,7 +264,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_line", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_line.rs"], test_options: { @@ -258,7 +274,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_to_end", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_to_end.rs"], test_options: { @@ -268,7 +284,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_to_string", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_to_string.rs"], test_options: { @@ -278,7 +294,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_until", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_until.rs"], test_options: { @@ -288,7 +304,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_split.rs"], test_options: { @@ -298,7 +314,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_take", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_take.rs"], test_options: { @@ -308,7 +324,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write.rs"], test_options: { @@ -318,7 +334,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write_all", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write_all.rs"], test_options: { @@ -328,7 +344,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write_buf", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write_buf.rs"], test_options: { @@ -338,7 +354,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write_int", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write_int.rs"], test_options: { @@ -348,7 +364,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_join", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_join.rs"], test_options: { @@ -358,7 +374,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_pin", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_pin.rs"], test_options: { @@ -368,7 +384,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_select", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_select.rs"], test_options: { @@ -378,7 +394,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_test", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_test.rs"], test_options: { @@ -388,7 +404,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_try_join", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_try_join.rs"], test_options: { @@ -398,7 +414,7 @@ rust_test { rust_test { name: "tokio_test_tests_net_bind_resource", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/net_bind_resource.rs"], test_options: { @@ -408,7 +424,7 @@ rust_test { rust_test { name: "tokio_test_tests_net_lookup_host", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/net_lookup_host.rs"], test_options: { @@ -418,7 +434,7 @@ rust_test { rust_test { name: "tokio_test_tests_no_rt", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/no_rt.rs"], test_options: { @@ -428,7 +444,7 @@ rust_test { rust_test { name: "tokio_test_tests_process_kill_on_drop", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/process_kill_on_drop.rs"], test_options: { @@ -438,7 +454,7 @@ rust_test { rust_test { name: "tokio_test_tests_rt_basic", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/rt_basic.rs"], test_options: { @@ -448,7 +464,7 @@ rust_test { rust_test { name: "tokio_test_tests_rt_common", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/rt_common.rs"], test_options: { @@ -458,7 +474,7 @@ rust_test { rust_test { name: "tokio_test_tests_rt_threaded", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/rt_threaded.rs"], test_options: { @@ -468,7 +484,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_barrier", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_barrier.rs"], test_options: { @@ -478,7 +494,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_broadcast", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_broadcast.rs"], test_options: { @@ -488,7 +504,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_errors", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_errors.rs"], test_options: { @@ -497,18 +513,8 @@ rust_test { } rust_test { - name: "tokio_test_tests_sync_mpsc", - defaults: ["tokio_defaults_tokio"], - host_supported: true, - srcs: ["tests/sync_mpsc.rs"], - test_options: { - unit_test: true, - }, -} - -rust_test { name: "tokio_test_tests_sync_mutex", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_mutex.rs"], test_options: { @@ -518,7 +524,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_mutex_owned", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_mutex_owned.rs"], test_options: { @@ -528,7 +534,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_notify", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_notify.rs"], test_options: { @@ -538,7 +544,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_oneshot", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_oneshot.rs"], test_options: { @@ -548,7 +554,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_rwlock", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_rwlock.rs"], test_options: { @@ -558,7 +564,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_semaphore", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_semaphore.rs"], test_options: { @@ -568,7 +574,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_semaphore_owned", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_semaphore_owned.rs"], test_options: { @@ -578,7 +584,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_watch", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_watch.rs"], test_options: { @@ -588,7 +594,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_abort", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_abort.rs"], test_options: { @@ -598,7 +604,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_blocking", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_blocking.rs"], test_options: { @@ -608,7 +614,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_local", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_local.rs"], test_options: { @@ -618,7 +624,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_local_set", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_local_set.rs"], test_options: { @@ -628,7 +634,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_accept", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_accept.rs"], test_options: { @@ -638,7 +644,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_connect", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_connect.rs"], test_options: { @@ -648,7 +654,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_echo", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_echo.rs"], test_options: { @@ -658,7 +664,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_into_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_into_split.rs"], test_options: { @@ -668,7 +674,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_into_std", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_into_std.rs"], test_options: { @@ -678,7 +684,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_peek", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_peek.rs"], test_options: { @@ -688,7 +694,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_shutdown", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_shutdown.rs"], test_options: { @@ -698,7 +704,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_socket", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_socket.rs"], test_options: { @@ -708,7 +714,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_split.rs"], test_options: { @@ -718,7 +724,7 @@ rust_test { rust_test { name: "tokio_test_tests_time_rt", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/time_rt.rs"], test_options: { @@ -728,7 +734,7 @@ rust_test { rust_test { name: "tokio_test_tests_udp", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/udp.rs"], test_options: { @@ -738,7 +744,7 @@ rust_test { rust_test { name: "tokio_test_tests_uds_cred", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/uds_cred.rs"], test_options: { @@ -748,7 +754,7 @@ rust_test { rust_test { name: "tokio_test_tests_uds_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/uds_split.rs"], test_options: { diff --git a/CHANGELOG.md b/CHANGELOG.md index afa8bf0..39a57fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,761 @@ +# 1.25.0 (January 28, 2023) + +### Fixed + +- rt: fix runtime metrics reporting ([#5330]) + +### Added + +- sync: add `broadcast::Sender::len` ([#5343]) + +### Changed + +- fs: increase maximum read buffer size to 2MiB ([#5397]) + +[#5330]: https://github.com/tokio-rs/tokio/pull/5330 +[#5343]: https://github.com/tokio-rs/tokio/pull/5343 +[#5397]: https://github.com/tokio-rs/tokio/pull/5397 + +# 1.24.2 (January 17, 2023) + +Forward ports 1.18.5 changes. + +### Fixed + +- io: fix unsoundness in `ReadHalf::unsplit` ([#5375]) + +[#5375]: https://github.com/tokio-rs/tokio/pull/5375 + +# 1.24.1 (January 6, 2022) + +This release fixes a compilation failure on targets without `AtomicU64` when using rustc older than 1.63. ([#5356]) + +[#5356]: https://github.com/tokio-rs/tokio/pull/5356 + +# 1.24.0 (January 5, 2022) + +### Fixed + - rt: improve native `AtomicU64` support detection ([#5284]) + +### Added + - rt: add configuration option for max number of I/O events polled from the OS + per tick ([#5186]) + - rt: add an environment variable for configuring the default number of worker + threads per runtime instance ([#4250]) + +### Changed + - sync: reduce MPSC channel stack usage ([#5294]) + - io: reduce lock contention in I/O operations ([#5300]) + - fs: speed up `read_dir()` by chunking operations ([#5309]) + - rt: use internal `ThreadId` implementation ([#5329]) + - test: don't auto-advance time when a `spawn_blocking` task is running ([#5115]) + +[#5186]: https://github.com/tokio-rs/tokio/pull/5186 +[#5294]: https://github.com/tokio-rs/tokio/pull/5294 +[#5284]: https://github.com/tokio-rs/tokio/pull/5284 +[#4250]: https://github.com/tokio-rs/tokio/pull/4250 +[#5300]: https://github.com/tokio-rs/tokio/pull/5300 +[#5329]: https://github.com/tokio-rs/tokio/pull/5329 +[#5115]: https://github.com/tokio-rs/tokio/pull/5115 +[#5309]: https://github.com/tokio-rs/tokio/pull/5309 + +# 1.23.1 (January 4, 2022) + +This release forward ports changes from 1.18.4. + +### Fixed + +- net: fix Windows named pipe server builder to maintain option when toggling + pipe mode ([#5336]). + +[#5336]: https://github.com/tokio-rs/tokio/pull/5336 + +# 1.23.0 (December 5, 2022) + +### Fixed + + - net: fix Windows named pipe connect ([#5208]) + - io: support vectored writes for `ChildStdin` ([#5216]) + - io: fix `async fn ready()` false positive for OS-specific events ([#5231]) + + ### Changed + - runtime: `yield_now` defers task until after driver poll ([#5223]) + - runtime: reduce amount of codegen needed per spawned task ([#5213]) + - windows: replace `winapi` dependency with `windows-sys` ([#5204]) + + [#5208]: https://github.com/tokio-rs/tokio/pull/5208 + [#5216]: https://github.com/tokio-rs/tokio/pull/5216 + [#5213]: https://github.com/tokio-rs/tokio/pull/5213 + [#5204]: https://github.com/tokio-rs/tokio/pull/5204 + [#5223]: https://github.com/tokio-rs/tokio/pull/5223 + [#5231]: https://github.com/tokio-rs/tokio/pull/5231 + +# 1.22.0 (November 17, 2022) + +### Added + - runtime: add `Handle::runtime_flavor` ([#5138]) + - sync: add `Mutex::blocking_lock_owned` ([#5130]) + - sync: add `Semaphore::MAX_PERMITS` ([#5144]) + - sync: add `merge()` to semaphore permits ([#4948]) + - sync: add `mpsc::WeakUnboundedSender` ([#5189]) + +### Added (unstable) + + - process: add `Command::process_group` ([#5114]) + - runtime: export metrics about the blocking thread pool ([#5161]) + - task: add `task::id()` and `task::try_id()` ([#5171]) + +### Fixed + - macros: don't take ownership of futures in macros ([#5087]) + - runtime: fix Stacked Borrows violation in `LocalOwnedTasks` ([#5099]) + - runtime: mitigate ABA with 32-bit queue indices when possible ([#5042]) + - task: wake local tasks to the local queue when woken by the same thread ([#5095]) + - time: panic in release mode when `mark_pending` called illegally ([#5093]) + - runtime: fix typo in expect message ([#5169]) + - runtime: fix `unsync_load` on atomic types ([#5175]) + - task: elaborate safety comments in task deallocation ([#5172]) + - runtime: fix `LocalSet` drop in thread local ([#5179]) + - net: remove libc type leakage in a public API ([#5191]) + - runtime: update the alignment of `CachePadded` ([#5106]) + +### Changed + - io: make `tokio::io::copy` continue filling the buffer when writer stalls ([#5066]) + - runtime: remove `coop::budget` from `LocalSet::run_until` ([#5155]) + - sync: make `Notify` panic safe ([#5154]) + +### Documented + - io: fix doc for `write_i8` to use signed integers ([#5040]) + - net: fix doc typos for TCP and UDP `set_tos` methods ([#5073]) + - net: fix function name in `UdpSocket::recv` documentation ([#5150]) + - sync: typo in `TryLockError` for `RwLock::try_write` ([#5160]) + - task: document that spawned tasks execute immediately ([#5117]) + - time: document return type of `timeout` ([#5118]) + - time: document that `timeout` checks only before poll ([#5126]) + - sync: specify return type of `oneshot::Receiver` in docs ([#5198]) + +### Internal changes + - runtime: use const `Mutex::new` for globals ([#5061]) + - runtime: remove `Option` around `mio::Events` in io driver ([#5078]) + - runtime: remove a conditional compilation clause ([#5104]) + - runtime: remove a reference to internal time handle ([#5107]) + - runtime: misc time driver cleanup ([#5120]) + - runtime: move signal driver to runtime module ([#5121]) + - runtime: signal driver now uses I/O driver directly ([#5125]) + - runtime: start decoupling I/O driver and I/O handle ([#5127]) + - runtime: switch `io::handle` refs with scheduler:Handle ([#5128]) + - runtime: remove Arc from I/O driver ([#5134]) + - runtime: use signal driver handle via `scheduler::Handle` ([#5135]) + - runtime: move internal clock fns out of context ([#5139]) + - runtime: remove `runtime::context` module ([#5140]) + - runtime: keep driver cfgs in `driver.rs` ([#5141]) + - runtime: add `runtime::context` to unify thread-locals ([#5143]) + - runtime: rename some confusing internal variables/fns ([#5151]) + - runtime: move `coop` mod into `runtime` ([#5152]) + - runtime: move budget state to context thread-local ([#5157]) + - runtime: move park logic into runtime module ([#5158]) + - runtime: move `Runtime` into its own file ([#5159]) + - runtime: unify entering a runtime with `Handle::enter` ([#5163]) + - runtime: remove handle reference from each scheduler ([#5166]) + - runtime: move `enter` into `context` ([#5167]) + - runtime: combine context and entered thread-locals ([#5168]) + - runtime: fix accidental unsetting of current handle ([#5178]) + - runtime: move `CoreStage` methods to `Core` ([#5182]) + - sync: name mpsc semaphore types ([#5146]) + +[#4948]: https://github.com/tokio-rs/tokio/pull/4948 +[#5040]: https://github.com/tokio-rs/tokio/pull/5040 +[#5042]: https://github.com/tokio-rs/tokio/pull/5042 +[#5061]: https://github.com/tokio-rs/tokio/pull/5061 +[#5066]: https://github.com/tokio-rs/tokio/pull/5066 +[#5073]: https://github.com/tokio-rs/tokio/pull/5073 +[#5078]: https://github.com/tokio-rs/tokio/pull/5078 +[#5087]: https://github.com/tokio-rs/tokio/pull/5087 +[#5093]: https://github.com/tokio-rs/tokio/pull/5093 +[#5095]: https://github.com/tokio-rs/tokio/pull/5095 +[#5099]: https://github.com/tokio-rs/tokio/pull/5099 +[#5104]: https://github.com/tokio-rs/tokio/pull/5104 +[#5106]: https://github.com/tokio-rs/tokio/pull/5106 +[#5107]: https://github.com/tokio-rs/tokio/pull/5107 +[#5114]: https://github.com/tokio-rs/tokio/pull/5114 +[#5117]: https://github.com/tokio-rs/tokio/pull/5117 +[#5118]: https://github.com/tokio-rs/tokio/pull/5118 +[#5120]: https://github.com/tokio-rs/tokio/pull/5120 +[#5121]: https://github.com/tokio-rs/tokio/pull/5121 +[#5125]: https://github.com/tokio-rs/tokio/pull/5125 +[#5126]: https://github.com/tokio-rs/tokio/pull/5126 +[#5127]: https://github.com/tokio-rs/tokio/pull/5127 +[#5128]: https://github.com/tokio-rs/tokio/pull/5128 +[#5130]: https://github.com/tokio-rs/tokio/pull/5130 +[#5134]: https://github.com/tokio-rs/tokio/pull/5134 +[#5135]: https://github.com/tokio-rs/tokio/pull/5135 +[#5138]: https://github.com/tokio-rs/tokio/pull/5138 +[#5138]: https://github.com/tokio-rs/tokio/pull/5138 +[#5139]: https://github.com/tokio-rs/tokio/pull/5139 +[#5140]: https://github.com/tokio-rs/tokio/pull/5140 +[#5141]: https://github.com/tokio-rs/tokio/pull/5141 +[#5143]: https://github.com/tokio-rs/tokio/pull/5143 +[#5144]: https://github.com/tokio-rs/tokio/pull/5144 +[#5144]: https://github.com/tokio-rs/tokio/pull/5144 +[#5146]: https://github.com/tokio-rs/tokio/pull/5146 +[#5150]: https://github.com/tokio-rs/tokio/pull/5150 +[#5151]: https://github.com/tokio-rs/tokio/pull/5151 +[#5152]: https://github.com/tokio-rs/tokio/pull/5152 +[#5154]: https://github.com/tokio-rs/tokio/pull/5154 +[#5155]: https://github.com/tokio-rs/tokio/pull/5155 +[#5157]: https://github.com/tokio-rs/tokio/pull/5157 +[#5158]: https://github.com/tokio-rs/tokio/pull/5158 +[#5159]: https://github.com/tokio-rs/tokio/pull/5159 +[#5160]: https://github.com/tokio-rs/tokio/pull/5160 +[#5161]: https://github.com/tokio-rs/tokio/pull/5161 +[#5163]: https://github.com/tokio-rs/tokio/pull/5163 +[#5166]: https://github.com/tokio-rs/tokio/pull/5166 +[#5167]: https://github.com/tokio-rs/tokio/pull/5167 +[#5168]: https://github.com/tokio-rs/tokio/pull/5168 +[#5169]: https://github.com/tokio-rs/tokio/pull/5169 +[#5171]: https://github.com/tokio-rs/tokio/pull/5171 +[#5172]: https://github.com/tokio-rs/tokio/pull/5172 +[#5175]: https://github.com/tokio-rs/tokio/pull/5175 +[#5178]: https://github.com/tokio-rs/tokio/pull/5178 +[#5179]: https://github.com/tokio-rs/tokio/pull/5179 +[#5182]: https://github.com/tokio-rs/tokio/pull/5182 +[#5189]: https://github.com/tokio-rs/tokio/pull/5189 +[#5191]: https://github.com/tokio-rs/tokio/pull/5191 +[#5198]: https://github.com/tokio-rs/tokio/pull/5198 + +# 1.21.2 (September 27, 2022) + +This release removes the dependency on the `once_cell` crate to restore the MSRV +of 1.21.x, which is the latest minor version at the time of release. ([#5048]) + +[#5048]: https://github.com/tokio-rs/tokio/pull/5048 + +# 1.21.1 (September 13, 2022) + +### Fixed + +- net: fix dependency resolution for socket2 ([#5000]) +- task: ignore failure to set TLS in `LocalSet` Drop ([#4976]) + +[#4976]: https://github.com/tokio-rs/tokio/pull/4976 +[#5000]: https://github.com/tokio-rs/tokio/pull/5000 + +# 1.21.0 (September 2, 2022) + +This release is the first release of Tokio to intentionally support WASM. The +`sync,macros,io-util,rt,time` features are stabilized on WASM. Additionally the +wasm32-wasi target is given unstable support for the `net` feature. + +### Added + +- net: add `device` and `bind_device` methods to TCP/UDP sockets ([#4882]) +- net: add `tos` and `set_tos` methods to TCP and UDP sockets ([#4877]) +- net: add security flags to named pipe `ServerOptions` ([#4845]) +- signal: add more windows signal handlers ([#4924]) +- sync: add `mpsc::Sender::max_capacity` method ([#4904]) +- sync: implement Weak version of `mpsc::Sender` ([#4595]) +- task: add `LocalSet::enter` ([#4765]) +- task: stabilize `JoinSet` and `AbortHandle` ([#4920]) +- tokio: add `track_caller` to public APIs ([#4805], [#4848], [#4852]) +- wasm: initial support for `wasm32-wasi` target ([#4716]) + +### Fixed + +- miri: improve miri compatibility by avoiding temporary references in `linked_list::Link` impls ([#4841]) +- signal: don't register write interest on signal pipe ([#4898]) +- sync: add `#[must_use]` to lock guards ([#4886]) +- sync: fix hang when calling `recv` on closed and reopened broadcast channel ([#4867]) +- task: propagate attributes on task-locals ([#4837]) + +### Changed + +- fs: change panic to error in `File::start_seek` ([#4897]) +- io: reduce syscalls in `poll_read` ([#4840]) +- process: use blocking threadpool for child stdio I/O ([#4824]) +- signal: make `SignalKind` methods const ([#4956]) + +### Internal changes + +- rt: extract `basic_scheduler::Config` ([#4935]) +- rt: move I/O driver into `runtime` module ([#4942]) +- rt: rename internal scheduler types ([#4945]) + +### Documented + +- chore: fix typos and grammar ([#4858], [#4894], [#4928]) +- io: fix typo in `AsyncSeekExt::rewind` docs ([#4893]) +- net: add documentation to `try_read()` for zero-length buffers ([#4937]) +- runtime: remove incorrect panic section for `Builder::worker_threads` ([#4849]) +- sync: doc of `watch::Sender::send` improved ([#4959]) +- task: add cancel safety docs to `JoinHandle` ([#4901]) +- task: expand on cancellation of `spawn_blocking` ([#4811]) +- time: clarify that the first tick of `Interval::tick` happens immediately ([#4951]) + +### Unstable + +- rt: add unstable option to disable the LIFO slot ([#4936]) +- task: fix incorrect signature in `Builder::spawn_on` ([#4953]) +- task: make `task::Builder::spawn*` methods fallible ([#4823]) + +[#4595]: https://github.com/tokio-rs/tokio/pull/4595 +[#4716]: https://github.com/tokio-rs/tokio/pull/4716 +[#4765]: https://github.com/tokio-rs/tokio/pull/4765 +[#4805]: https://github.com/tokio-rs/tokio/pull/4805 +[#4811]: https://github.com/tokio-rs/tokio/pull/4811 +[#4823]: https://github.com/tokio-rs/tokio/pull/4823 +[#4824]: https://github.com/tokio-rs/tokio/pull/4824 +[#4837]: https://github.com/tokio-rs/tokio/pull/4837 +[#4840]: https://github.com/tokio-rs/tokio/pull/4840 +[#4841]: https://github.com/tokio-rs/tokio/pull/4841 +[#4845]: https://github.com/tokio-rs/tokio/pull/4845 +[#4848]: https://github.com/tokio-rs/tokio/pull/4848 +[#4849]: https://github.com/tokio-rs/tokio/pull/4849 +[#4852]: https://github.com/tokio-rs/tokio/pull/4852 +[#4858]: https://github.com/tokio-rs/tokio/pull/4858 +[#4867]: https://github.com/tokio-rs/tokio/pull/4867 +[#4877]: https://github.com/tokio-rs/tokio/pull/4877 +[#4882]: https://github.com/tokio-rs/tokio/pull/4882 +[#4886]: https://github.com/tokio-rs/tokio/pull/4886 +[#4893]: https://github.com/tokio-rs/tokio/pull/4893 +[#4894]: https://github.com/tokio-rs/tokio/pull/4894 +[#4897]: https://github.com/tokio-rs/tokio/pull/4897 +[#4898]: https://github.com/tokio-rs/tokio/pull/4898 +[#4901]: https://github.com/tokio-rs/tokio/pull/4901 +[#4904]: https://github.com/tokio-rs/tokio/pull/4904 +[#4920]: https://github.com/tokio-rs/tokio/pull/4920 +[#4924]: https://github.com/tokio-rs/tokio/pull/4924 +[#4928]: https://github.com/tokio-rs/tokio/pull/4928 +[#4935]: https://github.com/tokio-rs/tokio/pull/4935 +[#4936]: https://github.com/tokio-rs/tokio/pull/4936 +[#4937]: https://github.com/tokio-rs/tokio/pull/4937 +[#4942]: https://github.com/tokio-rs/tokio/pull/4942 +[#4945]: https://github.com/tokio-rs/tokio/pull/4945 +[#4951]: https://github.com/tokio-rs/tokio/pull/4951 +[#4953]: https://github.com/tokio-rs/tokio/pull/4953 +[#4956]: https://github.com/tokio-rs/tokio/pull/4956 +[#4959]: https://github.com/tokio-rs/tokio/pull/4959 + +# 1.20.4 (January 17, 2023) + +Forward ports 1.18.5 changes. + +### Fixed + +- io: fix unsoundness in `ReadHalf::unsplit` ([#5375]) + +[#5375]: https://github.com/tokio-rs/tokio/pull/5375 + +# 1.20.3 (January 3, 2022) + +This release forward ports changes from 1.18.4. + +### Fixed + +- net: fix Windows named pipe server builder to maintain option when toggling + pipe mode ([#5336]). + +[#5336]: https://github.com/tokio-rs/tokio/pull/5336 + +# 1.20.2 (September 27, 2022) + +This release removes the dependency on the `once_cell` crate to restore the MSRV +of the 1.20.x LTS release. ([#5048]) + +[#5048]: https://github.com/tokio-rs/tokio/pull/5048 + +# 1.20.1 (July 25, 2022) + +### Fixed + +- chore: fix version detection in build script ([#4860]) + +[#4860]: https://github.com/tokio-rs/tokio/pull/4860 + +# 1.20.0 (July 12, 2022) + +### Added +- tokio: add `track_caller` to public APIs ([#4772], [#4791], [#4793], [#4806], [#4808]) +- sync: Add `has_changed` method to `watch::Ref` ([#4758]) + +### Changed + +- time: remove `src/time/driver/wheel/stack.rs` ([#4766]) +- rt: clean up arguments passed to basic scheduler ([#4767]) +- net: be more specific about winapi features ([#4764]) +- tokio: use const initialized thread locals where possible ([#4677]) +- task: various small improvements to LocalKey ([#4795]) + +### Documented + +- fs: warn about performance pitfall ([#4762]) +- chore: fix spelling ([#4769]) +- sync: document spurious failures in oneshot ([#4777]) +- sync: add warning for watch in non-Send futures ([#4741]) +- chore: fix typo ([#4798]) + +### Unstable + +- joinset: rename `join_one` to `join_next` ([#4755]) +- rt: unhandled panic config for current thread rt ([#4770]) + +[#4677]: https://github.com/tokio-rs/tokio/pull/4677 +[#4741]: https://github.com/tokio-rs/tokio/pull/4741 +[#4755]: https://github.com/tokio-rs/tokio/pull/4755 +[#4758]: https://github.com/tokio-rs/tokio/pull/4758 +[#4762]: https://github.com/tokio-rs/tokio/pull/4762 +[#4764]: https://github.com/tokio-rs/tokio/pull/4764 +[#4766]: https://github.com/tokio-rs/tokio/pull/4766 +[#4767]: https://github.com/tokio-rs/tokio/pull/4767 +[#4769]: https://github.com/tokio-rs/tokio/pull/4769 +[#4770]: https://github.com/tokio-rs/tokio/pull/4770 +[#4772]: https://github.com/tokio-rs/tokio/pull/4772 +[#4777]: https://github.com/tokio-rs/tokio/pull/4777 +[#4791]: https://github.com/tokio-rs/tokio/pull/4791 +[#4793]: https://github.com/tokio-rs/tokio/pull/4793 +[#4795]: https://github.com/tokio-rs/tokio/pull/4795 +[#4798]: https://github.com/tokio-rs/tokio/pull/4798 +[#4806]: https://github.com/tokio-rs/tokio/pull/4806 +[#4808]: https://github.com/tokio-rs/tokio/pull/4808 + +# 1.19.2 (June 6, 2022) + +This release fixes another bug in `Notified::enable`. ([#4751]) + +[#4751]: https://github.com/tokio-rs/tokio/pull/4751 + +# 1.19.1 (June 5, 2022) + +This release fixes a bug in `Notified::enable`. ([#4747]) + +[#4747]: https://github.com/tokio-rs/tokio/pull/4747 + +# 1.19.0 (June 3, 2022) + +### Added + +- runtime: add `is_finished` method for `JoinHandle` and `AbortHandle` ([#4709]) +- runtime: make global queue and event polling intervals configurable ([#4671]) +- sync: add `Notified::enable` ([#4705]) +- sync: add `watch::Sender::send_if_modified` ([#4591]) +- sync: add resubscribe method to broadcast::Receiver ([#4607]) +- net: add `take_error` to `TcpSocket` and `TcpStream` ([#4739]) + +### Changed + +- io: refactor out usage of Weak in the io handle ([#4656]) + +### Fixed + +- macros: avoid starvation in `join!` and `try_join!` ([#4624]) + +### Documented + +- runtime: clarify semantics of tasks outliving `block_on` ([#4729]) +- time: fix example for `MissedTickBehavior::Burst` ([#4713]) + +### Unstable + +- metrics: correctly update atomics in `IoDriverMetrics` ([#4725]) +- metrics: fix compilation with unstable, process, and rt, but without net ([#4682]) +- task: add `#[track_caller]` to `JoinSet`/`JoinMap` ([#4697]) +- task: add `Builder::{spawn_on, spawn_local_on, spawn_blocking_on}` ([#4683]) +- task: add `consume_budget` for cooperative scheduling ([#4498]) +- task: add `join_set::Builder` for configuring `JoinSet` tasks ([#4687]) +- task: update return value of `JoinSet::join_one` ([#4726]) + +[#4498]: https://github.com/tokio-rs/tokio/pull/4498 +[#4591]: https://github.com/tokio-rs/tokio/pull/4591 +[#4607]: https://github.com/tokio-rs/tokio/pull/4607 +[#4624]: https://github.com/tokio-rs/tokio/pull/4624 +[#4656]: https://github.com/tokio-rs/tokio/pull/4656 +[#4671]: https://github.com/tokio-rs/tokio/pull/4671 +[#4682]: https://github.com/tokio-rs/tokio/pull/4682 +[#4683]: https://github.com/tokio-rs/tokio/pull/4683 +[#4687]: https://github.com/tokio-rs/tokio/pull/4687 +[#4697]: https://github.com/tokio-rs/tokio/pull/4697 +[#4705]: https://github.com/tokio-rs/tokio/pull/4705 +[#4709]: https://github.com/tokio-rs/tokio/pull/4709 +[#4713]: https://github.com/tokio-rs/tokio/pull/4713 +[#4725]: https://github.com/tokio-rs/tokio/pull/4725 +[#4726]: https://github.com/tokio-rs/tokio/pull/4726 +[#4729]: https://github.com/tokio-rs/tokio/pull/4729 +[#4739]: https://github.com/tokio-rs/tokio/pull/4739 + +# 1.18.5 (January 17, 2023) + +### Fixed + +- io: fix unsoundness in `ReadHalf::unsplit` ([#5375]) + +[#5375]: https://github.com/tokio-rs/tokio/pull/5375 + +# 1.18.4 (January 3, 2022) + +### Fixed + +- net: fix Windows named pipe server builder to maintain option when toggling + pipe mode ([#5336]). + +[#5336]: https://github.com/tokio-rs/tokio/pull/5336 + +# 1.18.3 (September 27, 2022) + +This release removes the dependency on the `once_cell` crate to restore the MSRV +of the 1.18.x LTS release. ([#5048]) + +[#5048]: https://github.com/tokio-rs/tokio/pull/5048 + +# 1.18.2 (May 5, 2022) + +Add missing features for the `winapi` dependency. ([#4663]) + +[#4663]: https://github.com/tokio-rs/tokio/pull/4663 + +# 1.18.1 (May 2, 2022) + +The 1.18.0 release broke the build for targets without 64-bit atomics when +building with `tokio_unstable`. This release fixes that. ([#4649]) + +[#4649]: https://github.com/tokio-rs/tokio/pull/4649 + +# 1.18.0 (April 27, 2022) + +This release adds a number of new APIs in `tokio::net`, `tokio::signal`, and +`tokio::sync`. In addition, it adds new unstable APIs to `tokio::task` (`Id`s +for uniquely identifying a task, and `AbortHandle` for remotely cancelling a +task), as well as a number of bugfixes. + +### Fixed + +- blocking: add missing `#[track_caller]` for `spawn_blocking` ([#4616]) +- macros: fix `select` macro to process 64 branches ([#4519]) +- net: fix `try_io` methods not calling Mio's `try_io` internally ([#4582]) +- runtime: recover when OS fails to spawn a new thread ([#4485]) + +### Added + +- net: add `UdpSocket::peer_addr` ([#4611]) +- net: add `try_read_buf` method for named pipes ([#4626]) +- signal: add `SignalKind` `Hash`/`Eq` impls and `c_int` conversion ([#4540]) +- signal: add support for signals up to `SIGRTMAX` ([#4555]) +- sync: add `watch::Sender::send_modify` method ([#4310]) +- sync: add `broadcast::Receiver::len` method ([#4542]) +- sync: add `watch::Receiver::same_channel` method ([#4581]) +- sync: implement `Clone` for `RecvError` types ([#4560]) + +### Changed + +- update `mio` to 0.8.1 ([#4582]) +- macros: rename `tokio::select!`'s internal `util` module ([#4543]) +- runtime: use `Vec::with_capacity` when building runtime ([#4553]) + +### Documented + +- improve docs for `tokio_unstable` ([#4524]) +- runtime: include more documentation for thread_pool/worker ([#4511]) +- runtime: update `Handle::current`'s docs to mention `EnterGuard` ([#4567]) +- time: clarify platform specific timer resolution ([#4474]) +- signal: document that `Signal::recv` is cancel-safe ([#4634]) +- sync: `UnboundedReceiver` close docs ([#4548]) + +### Unstable + +The following changes only apply when building with `--cfg tokio_unstable`: + +- task: add `task::Id` type ([#4630]) +- task: add `AbortHandle` type for cancelling tasks in a `JoinSet` ([#4530], + [#4640]) +- task: fix missing `doc(cfg(...))` attributes for `JoinSet` ([#4531]) +- task: fix broken link in `AbortHandle` RustDoc ([#4545]) +- metrics: add initial IO driver metrics ([#4507]) + + +[#4616]: https://github.com/tokio-rs/tokio/pull/4616 +[#4519]: https://github.com/tokio-rs/tokio/pull/4519 +[#4582]: https://github.com/tokio-rs/tokio/pull/4582 +[#4485]: https://github.com/tokio-rs/tokio/pull/4485 +[#4613]: https://github.com/tokio-rs/tokio/pull/4613 +[#4611]: https://github.com/tokio-rs/tokio/pull/4611 +[#4626]: https://github.com/tokio-rs/tokio/pull/4626 +[#4540]: https://github.com/tokio-rs/tokio/pull/4540 +[#4555]: https://github.com/tokio-rs/tokio/pull/4555 +[#4310]: https://github.com/tokio-rs/tokio/pull/4310 +[#4542]: https://github.com/tokio-rs/tokio/pull/4542 +[#4581]: https://github.com/tokio-rs/tokio/pull/4581 +[#4560]: https://github.com/tokio-rs/tokio/pull/4560 +[#4631]: https://github.com/tokio-rs/tokio/pull/4631 +[#4582]: https://github.com/tokio-rs/tokio/pull/4582 +[#4543]: https://github.com/tokio-rs/tokio/pull/4543 +[#4553]: https://github.com/tokio-rs/tokio/pull/4553 +[#4524]: https://github.com/tokio-rs/tokio/pull/4524 +[#4511]: https://github.com/tokio-rs/tokio/pull/4511 +[#4567]: https://github.com/tokio-rs/tokio/pull/4567 +[#4474]: https://github.com/tokio-rs/tokio/pull/4474 +[#4634]: https://github.com/tokio-rs/tokio/pull/4634 +[#4548]: https://github.com/tokio-rs/tokio/pull/4548 +[#4630]: https://github.com/tokio-rs/tokio/pull/4630 +[#4530]: https://github.com/tokio-rs/tokio/pull/4530 +[#4640]: https://github.com/tokio-rs/tokio/pull/4640 +[#4531]: https://github.com/tokio-rs/tokio/pull/4531 +[#4545]: https://github.com/tokio-rs/tokio/pull/4545 +[#4507]: https://github.com/tokio-rs/tokio/pull/4507 + +# 1.17.0 (February 16, 2022) + +This release updates the minimum supported Rust version (MSRV) to 1.49, the +`mio` dependency to v0.8, and the (optional) `parking_lot` dependency to v0.12. +Additionally, it contains several bug fixes, as well as internal refactoring and +performance improvements. + +### Fixed + +- time: prevent panicking in `sleep` with large durations ([#4495]) +- time: eliminate potential panics in `Instant` arithmetic on platforms where + `Instant::now` is not monotonic ([#4461]) +- io: fix `DuplexStream` not participating in cooperative yielding ([#4478]) +- rt: fix potential double panic when dropping a `JoinHandle` ([#4430]) + +### Changed + +- update minimum supported Rust version to 1.49 ([#4457]) +- update `parking_lot` dependency to v0.12.0 ([#4459]) +- update `mio` dependency to v0.8 ([#4449]) +- rt: remove an unnecessary lock in the blocking pool ([#4436]) +- rt: remove an unnecessary enum in the basic scheduler ([#4462]) +- time: use bit manipulation instead of modulo to improve performance ([#4480]) +- net: use `std::future::Ready` instead of our own `Ready` future ([#4271]) +- replace deprecated `atomic::spin_loop_hint` with `hint::spin_loop` ([#4491]) +- fix miri failures in intrusive linked lists ([#4397]) + +### Documented + +- io: add an example for `tokio::process::ChildStdin` ([#4479]) + +### Unstable + +The following changes only apply when building with `--cfg tokio_unstable`: + +- task: fix missing location information in `tracing` spans generated by + `spawn_local` ([#4483]) +- task: add `JoinSet` for managing sets of tasks ([#4335]) +- metrics: fix compilation error on MIPS ([#4475]) +- metrics: fix compilation error on arm32v7 ([#4453]) + +[#4495]: https://github.com/tokio-rs/tokio/pull/4495 +[#4461]: https://github.com/tokio-rs/tokio/pull/4461 +[#4478]: https://github.com/tokio-rs/tokio/pull/4478 +[#4430]: https://github.com/tokio-rs/tokio/pull/4430 +[#4457]: https://github.com/tokio-rs/tokio/pull/4457 +[#4459]: https://github.com/tokio-rs/tokio/pull/4459 +[#4449]: https://github.com/tokio-rs/tokio/pull/4449 +[#4462]: https://github.com/tokio-rs/tokio/pull/4462 +[#4436]: https://github.com/tokio-rs/tokio/pull/4436 +[#4480]: https://github.com/tokio-rs/tokio/pull/4480 +[#4271]: https://github.com/tokio-rs/tokio/pull/4271 +[#4491]: https://github.com/tokio-rs/tokio/pull/4491 +[#4397]: https://github.com/tokio-rs/tokio/pull/4397 +[#4479]: https://github.com/tokio-rs/tokio/pull/4479 +[#4483]: https://github.com/tokio-rs/tokio/pull/4483 +[#4335]: https://github.com/tokio-rs/tokio/pull/4335 +[#4475]: https://github.com/tokio-rs/tokio/pull/4475 +[#4453]: https://github.com/tokio-rs/tokio/pull/4453 + +# 1.16.1 (January 28, 2022) + +This release fixes a bug in [#4428] with the change [#4437]. + +[#4428]: https://github.com/tokio-rs/tokio/pull/4428 +[#4437]: https://github.com/tokio-rs/tokio/pull/4437 + +# 1.16.0 (January 27, 2022) + +Fixes a soundness bug in `io::Take` ([#4428]). The unsoundness is exposed when +leaking memory in the given `AsyncRead` implementation and then overwriting the +supplied buffer: + +```rust +impl AsyncRead for Buggy { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_> + ) -> Poll<Result<()>> { + let new_buf = vec![0; 5].leak(); + *buf = ReadBuf::new(new_buf); + buf.put_slice(b"hello"); + Poll::Ready(Ok(())) + } +} +``` + +Also, this release includes improvements to the multi-threaded scheduler that +can increase throughput by up to 20% in some cases ([#4383]). + +### Fixed + +- io: **soundness** don't expose uninitialized memory when using `io::Take` in edge case ([#4428]) +- fs: ensure `File::write` results in a `write` syscall when the runtime shuts down ([#4316]) +- process: drop pipe after child exits in `wait_with_output` ([#4315]) +- rt: improve error message when spawning a thread fails ([#4398]) +- rt: reduce false-positive thread wakups in the multi-threaded scheduler ([#4383]) +- sync: don't inherit `Send` from `parking_lot::*Guard` ([#4359]) + +### Added + +- net: `TcpSocket::linger()` and `set_linger()` ([#4324]) +- net: impl `UnwindSafe` for socket types ([#4384]) +- rt: impl `UnwindSafe` for `JoinHandle` ([#4418]) +- sync: `watch::Receiver::has_changed()` ([#4342]) +- sync: `oneshot::Receiver::blocking_recv()` ([#4334]) +- sync: `RwLock` blocking operations ([#4425]) + +### Unstable + +The following changes only apply when building with `--cfg tokio_unstable` + +- rt: **breaking change** overhaul runtime metrics API ([#4373]) + +[#4428]: https://github.com/tokio-rs/tokio/pull/4428 +[#4316]: https://github.com/tokio-rs/tokio/pull/4316 +[#4315]: https://github.com/tokio-rs/tokio/pull/4315 +[#4398]: https://github.com/tokio-rs/tokio/pull/4398 +[#4383]: https://github.com/tokio-rs/tokio/pull/4383 +[#4359]: https://github.com/tokio-rs/tokio/pull/4359 +[#4324]: https://github.com/tokio-rs/tokio/pull/4324 +[#4384]: https://github.com/tokio-rs/tokio/pull/4384 +[#4418]: https://github.com/tokio-rs/tokio/pull/4418 +[#4342]: https://github.com/tokio-rs/tokio/pull/4342 +[#4334]: https://github.com/tokio-rs/tokio/pull/4334 +[#4425]: https://github.com/tokio-rs/tokio/pull/4425 +[#4373]: https://github.com/tokio-rs/tokio/pull/4373 + +# 1.15.0 (December 15, 2021) + +### Fixed + +- io: add cooperative yielding support to `io::empty()` ([#4300]) +- time: make timeout robust against budget-depleting tasks ([#4314]) + +### Changed + +- update minimum supported Rust version to 1.46. + +### Added + +- time: add `Interval::reset()` ([#4248]) +- io: add explicit lifetimes to `AsyncFdReadyGuard` ([#4267]) +- process: add `Command::as_std()` ([#4295]) + +### Added (unstable) + +- tracing: instrument `tokio::sync` types ([#4302]) + +[#4302]: https://github.com/tokio-rs/tokio/pull/4302 +[#4300]: https://github.com/tokio-rs/tokio/pull/4300 +[#4295]: https://github.com/tokio-rs/tokio/pull/4295 +[#4267]: https://github.com/tokio-rs/tokio/pull/4267 +[#4248]: https://github.com/tokio-rs/tokio/pull/4248 +[#4314]: https://github.com/tokio-rs/tokio/pull/4314 + # 1.14.0 (November 15, 2021) ### Fixed @@ -919,7 +1677,7 @@ Biggest changes are: - Feature flags are simplified - `rt-core` and `rt-util` are combined to `rt` - `rt-threaded` is renamed to `rt-multi-thread` to match builder API - - `tcp`, `udp`, `uds`, `dns` are combied to `net`. + - `tcp`, `udp`, `uds`, `dns` are combined to `net`. - `parking_lot` is included with `full` ### Changes @@ -1417,7 +2175,7 @@ Biggest changes are: - `net::lookup_host` maps a `T: ToSocketAddrs` to a stream of `SocketAddrs` ([#1870]). - `process::Child` fields are made public to match `std` ([#2014]). - impl `Stream` for `sync::broadcast::Receiver` ([#2012]). -- `sync::RwLock` provides an asynchonous read-write lock ([#1699]). +- `sync::RwLock` provides an asynchronous read-write lock ([#1699]). - `runtime::Handle::current` returns the handle for the current runtime ([#2040]). - `StreamExt::filter` filters stream values according to a predicate ([#2001]). - `StreamExt::filter_map` simultaneously filter and map stream values ([#2001]). @@ -1526,7 +2284,7 @@ Biggest changes are: ### Fixes - calling `spawn_blocking` after runtime shutdown ([#1875]). -- `LocalSet` drop inifinite loop ([#1892]). +- `LocalSet` drop infinite loop ([#1892]). - `LocalSet` hang under load ([#1905]). - improved documentation ([#1865], [#1866], [#1868], [#1874], [#1876], [#1911]). @@ -11,23 +11,48 @@ [package] edition = "2018" +rust-version = "1.49" name = "tokio" -version = "1.14.0" +version = "1.25.0" authors = ["Tokio Contributors <team@tokio.rs>"] -description = "An event-driven, non-blocking I/O platform for writing asynchronous I/O\nbacked applications.\n" +description = """ +An event-driven, non-blocking I/O platform for writing asynchronous I/O +backed applications. +""" homepage = "https://tokio.rs" -documentation = "https://docs.rs/tokio/1.14.0/tokio/" readme = "README.md" -keywords = ["io", "async", "non-blocking", "futures"] -categories = ["asynchronous", "network-programming"] +keywords = [ + "io", + "async", + "non-blocking", + "futures", +] +categories = [ + "asynchronous", + "network-programming", +] license = "MIT" repository = "https://github.com/tokio-rs/tokio" + [package.metadata.docs.rs] all-features = true -rustdoc-args = ["--cfg", "docsrs"] +rustdoc-args = [ + "--cfg", + "docsrs", + "--cfg", + "tokio_unstable", +] +rustc-args = [ + "--cfg", + "tokio_unstable", +] [package.metadata.playground] -features = ["full", "test-util"] +features = [ + "full", + "test-util", +] + [dependencies.bytes] version = "1.0.0" optional = true @@ -37,27 +62,24 @@ version = "2.2" optional = true [dependencies.mio] -version = "0.7.6" +version = "0.8.4" optional = true [dependencies.num_cpus] version = "1.8.0" optional = true -[dependencies.once_cell] -version = "1.5.2" -optional = true - [dependencies.parking_lot] -version = "0.11.0" +version = "0.12.0" optional = true [dependencies.pin-project-lite] version = "0.2.0" [dependencies.tokio-macros] -version = "1.6.0" +version = "1.7.0" optional = true + [dev-dependencies.async-stream] version = "0.3" @@ -66,16 +88,7 @@ version = "0.3.0" features = ["async-await"] [dev-dependencies.mockall] -version = "0.10.2" - -[dev-dependencies.proptest] -version = "1" - -[dev-dependencies.rand] -version = "0.8.0" - -[dev-dependencies.socket2] -version = "0.4" +version = "0.11.1" [dev-dependencies.tempfile] version = "3.1.0" @@ -85,36 +98,120 @@ version = "0.1" [dev-dependencies.tokio-test] version = "0.4.0" + [build-dependencies.autocfg] -version = "1" +version = "1.1" [features] default = [] fs = [] -full = ["fs", "io-util", "io-std", "macros", "net", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "sync", "time"] +full = [ + "fs", + "io-util", + "io-std", + "macros", + "net", + "parking_lot", + "process", + "rt", + "rt-multi-thread", + "signal", + "sync", + "time", +] io-std = [] -io-util = ["memchr", "bytes"] +io-util = [ + "memchr", + "bytes", +] macros = ["tokio-macros"] -net = ["libc", "mio/os-poll", "mio/os-util", "mio/tcp", "mio/udp", "mio/uds", "winapi/namedpipeapi"] -process = ["bytes", "once_cell", "libc", "mio/os-poll", "mio/os-util", "mio/uds", "signal-hook-registry", "winapi/threadpoollegacyapiset"] +net = [ + "libc", + "mio/os-poll", + "mio/os-ext", + "mio/net", + "socket2", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_Security", + "windows-sys/Win32_Storage_FileSystem", + "windows-sys/Win32_System_Pipes", + "windows-sys/Win32_System_SystemServices", +] +process = [ + "bytes", + "libc", + "mio/os-poll", + "mio/os-ext", + "mio/net", + "signal-hook-registry", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_System_Threading", + "windows-sys/Win32_System_WindowsProgramming", +] rt = [] -rt-multi-thread = ["num_cpus", "rt"] -signal = ["once_cell", "libc", "mio/os-poll", "mio/uds", "mio/os-util", "signal-hook-registry", "winapi/consoleapi"] +rt-multi-thread = [ + "num_cpus", + "rt", +] +signal = [ + "libc", + "mio/os-poll", + "mio/net", + "mio/os-ext", + "signal-hook-registry", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_System_Console", +] stats = [] sync = [] -test-util = ["rt", "sync", "time"] +test-util = [ + "rt", + "sync", + "time", +] time = [] + +[target."cfg(all(any(target_arch = \"wasm32\", target_arch = \"wasm64\"), not(target_os = \"wasi\")))".dev-dependencies.wasm-bindgen-test] +version = "0.3.0" + +[target."cfg(docsrs)".dependencies.windows-sys] +version = "0.42.0" +features = [ + "Win32_Foundation", + "Win32_Security_Authorization", +] + [target."cfg(loom)".dev-dependencies.loom] -version = "0.5" -features = ["futures", "checkpoint"] +version = "0.5.2" +features = [ + "futures", + "checkpoint", +] + +[target."cfg(not(all(any(target_arch = \"wasm32\", target_arch = \"wasm64\"), target_os = \"unknown\")))".dev-dependencies.rand] +version = "0.8.0" + +[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dependencies.socket2] +version = "0.4.4" +features = ["all"] +optional = true + +[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dev-dependencies.proptest] +version = "1" + +[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dev-dependencies.socket2] +version = "0.4" + [target."cfg(target_os = \"freebsd\")".dev-dependencies.mio-aio] -version = "0.6.0" +version = "0.7.0" features = ["tokio"] + [target."cfg(tokio_unstable)".dependencies.tracing] version = "0.1.25" features = ["std"] optional = true default-features = false + [target."cfg(unix)".dependencies.libc] version = "0.2.42" optional = true @@ -122,14 +219,21 @@ optional = true [target."cfg(unix)".dependencies.signal-hook-registry] version = "1.1.1" optional = true + [target."cfg(unix)".dev-dependencies.libc] version = "0.2.42" [target."cfg(unix)".dev-dependencies.nix] -version = "0.22.0" -[target."cfg(windows)".dependencies.winapi] -version = "0.3.8" -optional = true +version = "0.26" +features = [ + "fs", + "socket", +] default-features = false + +[target."cfg(windows)".dependencies.windows-sys] +version = "0.42.0" +optional = true + [target."cfg(windows)".dev-dependencies.ntapi] version = "0.3.6" diff --git a/Cargo.toml.orig b/Cargo.toml.orig index 348ec46..0f6d30a 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -3,16 +3,15 @@ name = "tokio" # When releasing to crates.io: # - Remove path dependencies # - Update doc url -# - Cargo.toml # - README.md # - Update CHANGELOG.md. -# - Create "v1.0.x" git tag. -version = "1.14.0" +# - Create "v1.x.y" git tag. +version = "1.25.0" edition = "2018" +rust-version = "1.49" authors = ["Tokio Contributors <team@tokio.rs>"] license = "MIT" readme = "README.md" -documentation = "https://docs.rs/tokio/1.14.0/tokio/" repository = "https://github.com/tokio-rs/tokio" homepage = "https://tokio.rs" description = """ @@ -47,25 +46,28 @@ io-util = ["memchr", "bytes"] # stdin, stdout, stderr io-std = [] macros = ["tokio-macros"] -stats = [] net = [ "libc", "mio/os-poll", - "mio/os-util", - "mio/tcp", - "mio/udp", - "mio/uds", - "winapi/namedpipeapi", + "mio/os-ext", + "mio/net", + "socket2", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_Security", + "windows-sys/Win32_Storage_FileSystem", + "windows-sys/Win32_System_Pipes", + "windows-sys/Win32_System_SystemServices", ] process = [ "bytes", - "once_cell", "libc", "mio/os-poll", - "mio/os-util", - "mio/uds", + "mio/os-ext", + "mio/net", "signal-hook-registry", - "winapi/threadpoollegacyapiset", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_System_Threading", + "windows-sys/Win32_System_WindowsProgramming", ] # Includes basic task execution capabilities rt = [] @@ -74,30 +76,40 @@ rt-multi-thread = [ "rt", ] signal = [ - "once_cell", "libc", "mio/os-poll", - "mio/uds", - "mio/os-util", + "mio/net", + "mio/os-ext", "signal-hook-registry", - "winapi/consoleapi", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_System_Console", ] sync = [] test-util = ["rt", "sync", "time"] time = [] +# Technically, removing this is a breaking change even though it only ever did +# anything with the unstable flag on. It is probably safe to get rid of it after +# a few releases. +stats = [] + +[build-dependencies] +autocfg = "1.1" + [dependencies] -tokio-macros = { version = "1.6.0", path = "../tokio-macros", optional = true } +tokio-macros = { version = "1.7.0", path = "../tokio-macros", optional = true } pin-project-lite = "0.2.0" # Everything else is optional... bytes = { version = "1.0.0", optional = true } -once_cell = { version = "1.5.2", optional = true } memchr = { version = "2.2", optional = true } -mio = { version = "0.7.6", optional = true } +mio = { version = "0.8.4", optional = true } num_cpus = { version = "1.8.0", optional = true } -parking_lot = { version = "0.11.0", optional = true } +parking_lot = { version = "0.12.0", optional = true } + +[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dependencies] +socket2 = { version = "0.4.4", optional = true, features = [ "all" ] } # Currently unstable. The API exposed by these features may be broken at any time. # Requires `--cfg tokio_unstable` to enable. @@ -110,13 +122,19 @@ signal-hook-registry = { version = "1.1.1", optional = true } [target.'cfg(unix)'.dev-dependencies] libc = { version = "0.2.42" } -nix = { version = "0.22.0" } +nix = { version = "0.26", default-features = false, features = ["fs", "socket"] } -[target.'cfg(windows)'.dependencies.winapi] -version = "0.3.8" -default-features = false +[target.'cfg(windows)'.dependencies.windows-sys] +version = "0.42.0" optional = true +[target.'cfg(docsrs)'.dependencies.windows-sys] +version = "0.42.0" +features = [ + "Win32_Foundation", + "Win32_Security_Authorization", +] + [target.'cfg(windows)'.dev-dependencies.ntapi] version = "0.3.6" @@ -124,25 +142,33 @@ version = "0.3.6" tokio-test = { version = "0.4.0", path = "../tokio-test" } tokio-stream = { version = "0.1", path = "../tokio-stream" } futures = { version = "0.3.0", features = ["async-await"] } -mockall = "0.10.2" -proptest = "1" -rand = "0.8.0" +mockall = "0.11.1" tempfile = "3.1.0" async-stream = "0.3" + +[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dev-dependencies] +proptest = "1" socket2 = "0.4" +[target.'cfg(not(all(any(target_arch = "wasm32", target_arch = "wasm64"), target_os = "unknown")))'.dev-dependencies] +rand = "0.8.0" + +[target.'cfg(all(any(target_arch = "wasm32", target_arch = "wasm64"), not(target_os = "wasi")))'.dev-dependencies] +wasm-bindgen-test = "0.3.0" + [target.'cfg(target_os = "freebsd")'.dev-dependencies] -mio-aio = { version = "0.6.0", features = ["tokio"] } +mio-aio = { version = "0.7.0", features = ["tokio"] } [target.'cfg(loom)'.dev-dependencies] -loom = { version = "0.5", features = ["futures", "checkpoint"] } - -[build-dependencies] -autocfg = "1" # Needed for conditionally enabling `track-caller` +loom = { version = "0.5.2", features = ["futures", "checkpoint"] } [package.metadata.docs.rs] all-features = true -rustdoc-args = ["--cfg", "docsrs"] +# enable unstable features in the documentation +rustdoc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"] +# it's necessary to _also_ pass `--cfg tokio_unstable` to rustc, or else +# dependencies will not be enabled, and the docs build will fail. +rustc-args = ["--cfg", "tokio_unstable"] [package.metadata.playground] features = ["full", "test-util"] @@ -1,4 +1,4 @@ -Copyright (c) 2021 Tokio Contributors +Copyright (c) 2023 Tokio Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated @@ -1,3 +1,7 @@ +# This project was upgraded with external_updater. +# Usage: tools/external_updater/updater.sh update rust/crates/tokio +# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md + name: "tokio" description: "An event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications." third_party { @@ -7,13 +11,13 @@ third_party { } url { type: ARCHIVE - value: "https://static.crates.io/crates/tokio/tokio-1.14.0.crate" + value: "https://static.crates.io/crates/tokio/tokio-1.25.0.crate" } - version: "1.14.0" + version: "1.25.0" license_type: NOTICE last_upgrade_date { - year: 2021 - month: 11 - day: 16 + year: 2023 + month: 2 + day: 6 } } @@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml: ```toml [dependencies] -tokio = { version = "1.14.0", features = ["full"] } +tokio = { version = "1.25.0", features = ["full"] } ``` Then, on your main.rs: @@ -161,11 +161,33 @@ several other libraries, including: [`mio`]: https://github.com/tokio-rs/mio [`bytes`]: https://github.com/tokio-rs/bytes +## Changelog + +The Tokio repository contains multiple crates. Each crate has its own changelog. + + * `tokio` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio/CHANGELOG.md) + * `tokio-util` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-util/CHANGELOG.md) + * `tokio-stream` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-stream/CHANGELOG.md) + * `tokio-macros` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-macros/CHANGELOG.md) + * `tokio-test` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-test/CHANGELOG.md) + ## Supported Rust Versions -Tokio is built against the latest stable release. The minimum supported version -is 1.45. The current Tokio version is not guaranteed to build on Rust versions -earlier than the minimum supported version. +<!-- +When updating this, also update: +- .github/workflows/ci.yml +- CONTRIBUTING.md +- README.md +- tokio/README.md +- tokio/Cargo.toml +- tokio-util/Cargo.toml +- tokio-test/Cargo.toml +- tokio-stream/Cargo.toml +--> + +Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at +least** 6 months. When increasing the MSRV, the new Rust version must have been +released at least six months ago. The current MSRV is 1.49.0. ## Release schedule @@ -180,17 +202,18 @@ warrants a patch release with a fix for the bug, it will be backported and released as a new patch release for each LTS minor version. Our current LTS releases are: - * `1.8.x` - LTS release until February 2022. + * `1.18.x` - LTS release until June 2023 + * `1.20.x` - LTS release until September 2023. -Each LTS release will continue to receive backported fixes for at least half a -year. If you wish to use a fixed minor release in your project, we recommend -that you use an LTS release. +Each LTS release will continue to receive backported fixes for at least a year. +If you wish to use a fixed minor release in your project, we recommend that you +use an LTS release. To use a fixed minor version, you can specify the version with a tilde. For -example, to specify that you wish to use the newest `1.8.x` patch release, you +example, to specify that you wish to use the newest `1.18.x` patch release, you can use the following dependency specification: ```text -tokio = { version = "~1.8", features = [...] } +tokio = { version = "~1.18", features = [...] } ``` ## License diff --git a/TEST_MAPPING b/TEST_MAPPING index 2e7046e..63733b6 100644 --- a/TEST_MAPPING +++ b/TEST_MAPPING @@ -2,20 +2,23 @@ { "imports": [ { + "path": "external/rust/crates/async-stream" + }, + { "path": "external/rust/crates/futures-util" }, { "path": "external/rust/crates/tokio-test" - } - ], - "presubmit": [ - { - "name": "doh_unit_test" }, { - "name": "rustBinderTest" + "path": "external/uwb/src" }, { + "path": "packages/modules/DnsResolver" + } + ], + "presubmit": [ + { "name": "tokio_test_tests__require_full" }, { @@ -133,9 +136,6 @@ "name": "tokio_test_tests_sync_errors" }, { - "name": "tokio_test_tests_sync_mpsc" - }, - { "name": "tokio_test_tests_sync_mutex" }, { @@ -213,12 +213,6 @@ ], "presubmit-rust": [ { - "name": "doh_unit_test" - }, - { - "name": "rustBinderTest" - }, - { "name": "tokio_test_tests__require_full" }, { @@ -336,9 +330,6 @@ "name": "tokio_test_tests_sync_errors" }, { - "name": "tokio_test_tests_sync_mpsc" - }, - { "name": "tokio_test_tests_sync_mutex" }, { @@ -1,11 +1,121 @@ use autocfg::AutoCfg; +const CONST_THREAD_LOCAL_PROBE: &str = r#" +{ + thread_local! { + static MY_PROBE: usize = const { 10 }; + } + + MY_PROBE.with(|val| *val) +} +"#; + +const ADDR_OF_PROBE: &str = r#" +{ + let my_var = 10; + ::std::ptr::addr_of!(my_var) +} +"#; + +const CONST_MUTEX_NEW_PROBE: &str = r#" +{ + static MY_MUTEX: ::std::sync::Mutex<i32> = ::std::sync::Mutex::new(1); + *MY_MUTEX.lock().unwrap() +} +"#; + +const TARGET_HAS_ATOMIC_PROBE: &str = r#" +{ + #[cfg(target_has_atomic = "ptr")] + let _ = (); +} +"#; + +const TARGET_ATOMIC_U64_PROBE: &str = r#" +{ + #[allow(unused_imports)] + use std::sync::atomic::AtomicU64 as _; +} +"#; + fn main() { + let mut enable_const_thread_local = false; + let mut enable_addr_of = false; + let mut enable_target_has_atomic = false; + let mut enable_const_mutex_new = false; + let mut target_needs_atomic_u64_fallback = false; + match AutoCfg::new() { Ok(ac) => { - // The #[track_caller] attribute was stabilized in rustc 1.46.0. - if ac.probe_rustc_version(1, 46) { - autocfg::emit("tokio_track_caller") + // These checks prefer to call only `probe_rustc_version` if that is + // enough to determine whether the feature is supported. This is + // because the `probe_expression` call involves a call to rustc, + // which the `probe_rustc_version` call avoids. + + // Const-initialized thread locals were stabilized in 1.59. + if ac.probe_rustc_version(1, 60) { + enable_const_thread_local = true; + } else if ac.probe_rustc_version(1, 59) { + // This compiler claims to be 1.59, but there are some nightly + // compilers that claim to be 1.59 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2021-12-06. + if ac.probe_expression(CONST_THREAD_LOCAL_PROBE) { + enable_const_thread_local = true; + } + } + + // The `addr_of` and `addr_of_mut` macros were stabilized in 1.51. + if ac.probe_rustc_version(1, 52) { + enable_addr_of = true; + } else if ac.probe_rustc_version(1, 51) { + // This compiler claims to be 1.51, but there are some nightly + // compilers that claim to be 1.51 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2021-01-31. + if ac.probe_expression(ADDR_OF_PROBE) { + enable_addr_of = true; + } + } + + // The `target_has_atomic` cfg was stabilized in 1.60. + if ac.probe_rustc_version(1, 61) { + enable_target_has_atomic = true; + } else if ac.probe_rustc_version(1, 60) { + // This compiler claims to be 1.60, but there are some nightly + // compilers that claim to be 1.60 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2022-02-11. + if ac.probe_expression(TARGET_HAS_ATOMIC_PROBE) { + enable_target_has_atomic = true; + } + } + + // If we can't tell using `target_has_atomic`, tell if the target + // has `AtomicU64` by trying to use it. + if !enable_target_has_atomic && !ac.probe_expression(TARGET_ATOMIC_U64_PROBE) { + target_needs_atomic_u64_fallback = true; + } + + // The `Mutex::new` method was made const in 1.63. + if ac.probe_rustc_version(1, 64) { + enable_const_mutex_new = true; + } else if ac.probe_rustc_version(1, 63) { + // This compiler claims to be 1.63, but there are some nightly + // compilers that claim to be 1.63 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2022-06-20. + if ac.probe_expression(CONST_MUTEX_NEW_PROBE) { + enable_const_mutex_new = true; + } } } @@ -19,4 +129,57 @@ fn main() { ); } } + + if !enable_const_thread_local { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_const_thread_local" + autocfg::emit("tokio_no_const_thread_local") + } + + if !enable_addr_of { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_addr_of" + autocfg::emit("tokio_no_addr_of") + } + + if !enable_target_has_atomic { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_target_has_atomic" + autocfg::emit("tokio_no_target_has_atomic") + } + + if !enable_const_mutex_new { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_const_mutex_new" + autocfg::emit("tokio_no_const_mutex_new") + } + + if target_needs_atomic_u64_fallback { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_atomic_u64" + autocfg::emit("tokio_no_atomic_u64") + } + + let target = ::std::env::var("TARGET").unwrap_or_default(); + + // We emit cfgs instead of using `target_family = "wasm"` that requires Rust 1.54. + // Note that these cfgs are unavailable in `Cargo.toml`. + if target.starts_with("wasm") { + autocfg::emit("tokio_wasm"); + if target.contains("wasi") { + autocfg::emit("tokio_wasi"); + } else { + autocfg::emit("tokio_wasm_not_wasi"); + } + } } diff --git a/cargo2android.json b/cargo2android.json index 75772f0..443e022 100644 --- a/cargo2android.json +++ b/cargo2android.json @@ -2,12 +2,12 @@ "add-toplevel-block": "cargo2android_tests.bp", "apex-available": [ "//apex_available:platform", - "com.android.bluetooth", + "com.android.btservices", "com.android.resolv", "com.android.uwb" ], "device": true, - "features": "io-util,macros,rt-multi-thread,sync,net,fs,time", + "features": "fs,io-util,macros,net,rt,rt-multi-thread,sync,time", "min-sdk-version": "29", "vendor-available": true, "run": true diff --git a/cargo2android_tests.bp b/cargo2android_tests.bp index 809c899..44bc4c4 100644 --- a/cargo2android_tests.bp +++ b/cargo2android_tests.bp @@ -1,5 +1,5 @@ rust_defaults { - name: "tokio_defaults_tokio", + name: "tokio_defaults_tests", crate_name: "tokio", cargo_env_compat: true, test_suites: ["general-tests"], @@ -43,7 +43,7 @@ rust_defaults { rust_test { name: "tokio_test_tests__require_full", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/_require_full.rs"], test_options: { @@ -53,7 +53,7 @@ rust_test { rust_test { name: "tokio_test_tests_buffered", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/buffered.rs"], test_options: { @@ -63,7 +63,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_async_fd", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_async_fd.rs"], test_options: { @@ -73,7 +73,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_async_read", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_async_read.rs"], test_options: { @@ -83,7 +83,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_chain", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_chain.rs"], test_options: { @@ -93,7 +93,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_copy", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_copy.rs"], test_options: { @@ -103,7 +103,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_copy_bidirectional", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_copy_bidirectional.rs"], test_options: { @@ -113,7 +113,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_driver", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_driver.rs"], test_options: { @@ -123,7 +123,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_driver_drop", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_driver_drop.rs"], test_options: { @@ -133,7 +133,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_lines", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_lines.rs"], test_options: { @@ -143,7 +143,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_mem_stream", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_mem_stream.rs"], test_options: { @@ -153,7 +153,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read.rs"], test_options: { @@ -163,7 +163,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_buf", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_buf.rs"], test_options: { @@ -173,7 +173,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_exact", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_exact.rs"], test_options: { @@ -183,7 +183,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_line", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_line.rs"], test_options: { @@ -193,7 +193,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_to_end", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_to_end.rs"], test_options: { @@ -203,7 +203,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_to_string", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_to_string.rs"], test_options: { @@ -213,7 +213,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_read_until", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_read_until.rs"], test_options: { @@ -223,7 +223,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_split.rs"], test_options: { @@ -233,7 +233,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_take", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_take.rs"], test_options: { @@ -243,7 +243,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write.rs"], test_options: { @@ -253,7 +253,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write_all", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write_all.rs"], test_options: { @@ -263,7 +263,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write_buf", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write_buf.rs"], test_options: { @@ -273,7 +273,7 @@ rust_test { rust_test { name: "tokio_test_tests_io_write_int", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/io_write_int.rs"], test_options: { @@ -283,7 +283,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_join", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_join.rs"], test_options: { @@ -293,7 +293,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_pin", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_pin.rs"], test_options: { @@ -303,7 +303,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_select", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_select.rs"], test_options: { @@ -313,7 +313,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_test", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_test.rs"], test_options: { @@ -323,7 +323,7 @@ rust_test { rust_test { name: "tokio_test_tests_macros_try_join", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/macros_try_join.rs"], test_options: { @@ -333,7 +333,7 @@ rust_test { rust_test { name: "tokio_test_tests_net_bind_resource", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/net_bind_resource.rs"], test_options: { @@ -343,7 +343,7 @@ rust_test { rust_test { name: "tokio_test_tests_net_lookup_host", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/net_lookup_host.rs"], test_options: { @@ -353,7 +353,7 @@ rust_test { rust_test { name: "tokio_test_tests_no_rt", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/no_rt.rs"], test_options: { @@ -363,7 +363,7 @@ rust_test { rust_test { name: "tokio_test_tests_process_kill_on_drop", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/process_kill_on_drop.rs"], test_options: { @@ -373,7 +373,7 @@ rust_test { rust_test { name: "tokio_test_tests_rt_basic", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/rt_basic.rs"], test_options: { @@ -383,7 +383,7 @@ rust_test { rust_test { name: "tokio_test_tests_rt_common", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/rt_common.rs"], test_options: { @@ -393,7 +393,7 @@ rust_test { rust_test { name: "tokio_test_tests_rt_threaded", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/rt_threaded.rs"], test_options: { @@ -403,7 +403,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_barrier", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_barrier.rs"], test_options: { @@ -413,7 +413,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_broadcast", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_broadcast.rs"], test_options: { @@ -423,7 +423,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_errors", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_errors.rs"], test_options: { @@ -432,18 +432,8 @@ rust_test { } rust_test { - name: "tokio_test_tests_sync_mpsc", - defaults: ["tokio_defaults_tokio"], - host_supported: true, - srcs: ["tests/sync_mpsc.rs"], - test_options: { - unit_test: true, - }, -} - -rust_test { name: "tokio_test_tests_sync_mutex", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_mutex.rs"], test_options: { @@ -453,7 +443,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_mutex_owned", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_mutex_owned.rs"], test_options: { @@ -463,7 +453,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_notify", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_notify.rs"], test_options: { @@ -473,7 +463,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_oneshot", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_oneshot.rs"], test_options: { @@ -483,7 +473,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_rwlock", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_rwlock.rs"], test_options: { @@ -493,7 +483,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_semaphore", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_semaphore.rs"], test_options: { @@ -503,7 +493,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_semaphore_owned", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_semaphore_owned.rs"], test_options: { @@ -513,7 +503,7 @@ rust_test { rust_test { name: "tokio_test_tests_sync_watch", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/sync_watch.rs"], test_options: { @@ -523,7 +513,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_abort", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_abort.rs"], test_options: { @@ -533,7 +523,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_blocking", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_blocking.rs"], test_options: { @@ -543,7 +533,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_local", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_local.rs"], test_options: { @@ -553,7 +543,7 @@ rust_test { rust_test { name: "tokio_test_tests_task_local_set", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/task_local_set.rs"], test_options: { @@ -563,7 +553,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_accept", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_accept.rs"], test_options: { @@ -573,7 +563,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_connect", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_connect.rs"], test_options: { @@ -583,7 +573,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_echo", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_echo.rs"], test_options: { @@ -593,7 +583,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_into_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_into_split.rs"], test_options: { @@ -603,7 +593,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_into_std", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_into_std.rs"], test_options: { @@ -613,7 +603,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_peek", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_peek.rs"], test_options: { @@ -623,7 +613,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_shutdown", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_shutdown.rs"], test_options: { @@ -633,7 +623,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_socket", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_socket.rs"], test_options: { @@ -643,7 +633,7 @@ rust_test { rust_test { name: "tokio_test_tests_tcp_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/tcp_split.rs"], test_options: { @@ -653,7 +643,7 @@ rust_test { rust_test { name: "tokio_test_tests_time_rt", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/time_rt.rs"], test_options: { @@ -663,7 +653,7 @@ rust_test { rust_test { name: "tokio_test_tests_udp", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/udp.rs"], test_options: { @@ -673,7 +663,7 @@ rust_test { rust_test { name: "tokio_test_tests_uds_cred", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/uds_cred.rs"], test_options: { @@ -683,7 +673,7 @@ rust_test { rust_test { name: "tokio_test_tests_uds_split", - defaults: ["tokio_defaults_tokio"], + defaults: ["tokio_defaults_tests"], host_supported: true, srcs: ["tests/uds_split.rs"], test_options: { diff --git a/docs/reactor-refactor.md b/docs/reactor-refactor.md index 1c9ace1..77e64f4 100644 --- a/docs/reactor-refactor.md +++ b/docs/reactor-refactor.md @@ -188,12 +188,12 @@ readiness, the driver's tick is packed into the atomic `usize`. The `ScheduledIo` readiness `AtomicUsize` is structured as: ``` -| reserved | generation | driver tick | readinesss | -|----------+------------+--------------+------------| -| 1 bit | 7 bits + 8 bits + 16 bits | +| shutdown | generation | driver tick | readiness | +|----------+------------+--------------+-----------| +| 1 bit | 7 bits + 8 bits + 16 bits | ``` -The `reserved` and `generation` components exist today. +The `shutdown` and `generation` components exist today. The `readiness()` function returns a `ReadyEvent` value. This value includes the `tick` component read with the resource's readiness value. When diff --git a/external-types.toml b/external-types.toml new file mode 100644 index 0000000..a5bde8e --- /dev/null +++ b/external-types.toml @@ -0,0 +1,11 @@ +# This config file is for the `cargo-check-external-types` tool that is run in CI. + +# The following are types that are allowed to be exposed in Tokio's public API. +# The standard library is allowed by default. +allowed_external_types = [ + "bytes::buf::buf_impl::Buf", + "bytes::buf::buf_mut::BufMut", + + "tokio_macros::*", +] + diff --git a/patches/Android.bp.patch b/patches/Android.bp.patch new file mode 100644 index 0000000..68fc808 --- /dev/null +++ b/patches/Android.bp.patch @@ -0,0 +1,39 @@ +diff --git a/Android.bp b/Android.bp +index 7d066e6..58422c6 100644 +--- a/Android.bp ++++ b/Android.bp +@@ -18,10 +18,9 @@ + ], + } + +-rust_library { +- name: "libtokio", ++rust_defaults { ++ name: "tokio_defaults", + host_supported: true, +- crate_name: "tokio", + cargo_env_compat: true, + cargo_pkg_version: "1.25.0", + srcs: ["src/lib.rs"], +@@ -64,6 +63,21 @@ + min_sdk_version: "29", + } + ++rust_library { ++ name: "libtokio", ++ crate_name: "tokio", ++ defaults: ["tokio_defaults"], ++} ++ ++rust_library { ++ name: "libtokio_for_test", ++ crate_name: "tokio", ++ defaults: ["tokio_defaults"], ++ features: [ ++ "test-util", ++ ], ++} ++ + rust_defaults { + name: "tokio_defaults_tests", + crate_name: "tokio", diff --git a/patches/macros_join.rs.patch b/patches/macros_join.rs.patch new file mode 100644 index 0000000..f041b14 --- /dev/null +++ b/patches/macros_join.rs.patch @@ -0,0 +1,12 @@ +diff --git a/tests/macros_join.rs b/tests/macros_join.rs +index 16e7c43..4441582 100644 +--- a/tests/macros_join.rs ++++ b/tests/macros_join.rs +@@ -66,6 +66,7 @@ async fn two_await() { + + #[test] + #[cfg(target_pointer_width = "64")] ++#[ignore = "Android: ignore until the compiler is updated. aliceryhl@ says these tests assume latest stable compiler."] + fn join_size() { + use futures::future; + use std::mem; diff --git a/patches/panic_unwind_tests.patch b/patches/panic_unwind_tests.patch new file mode 100644 index 0000000..c11065b --- /dev/null +++ b/patches/panic_unwind_tests.patch @@ -0,0 +1,53 @@ +diff --git a/patches/panic_unwind_tests.patch b/patches/panic_unwind_tests.patch +index adf431e..e69de29 100644 +--- a/patches/panic_unwind_tests.patch ++++ b/patches/panic_unwind_tests.patch +@@ -1,24 +0,0 @@ +-diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs +-index 9aa3484..53ee7d8 100644 +---- a/tests/sync_broadcast.rs +-+++ b/tests/sync_broadcast.rs +-@@ -292,6 +292,7 @@ fn capacity_too_big() { +- +- #[test] +-+#[cfg(panic = "unwind")] +- #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +- fn panic_in_clone() { +- use std::panic::{self, AssertUnwindSafe}; +- +-diff --git a/tests/sync_watch.rs b/tests/sync_watch.rs +-index 34f9b78..e8eacce 100644 +---- a/tests/sync_watch.rs +-+++ b/tests/sync_watch.rs +-@@ -214,6 +214,7 @@ fn reopened_after_subscribe() { +- +- #[test] +-+#[cfg(panic = "unwind")] +- #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +- fn send_modify_panic() { +- let (tx, mut rx) = watch::channel("one"); +- +diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs +index 67c378b..cd66924 100644 +--- a/tests/sync_broadcast.rs ++++ b/tests/sync_broadcast.rs +@@ -291,6 +291,7 @@ fn capacity_too_big() { + } + + #[test] ++#[cfg(panic = "unwind")] + #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding + fn panic_in_clone() { + use std::panic::{self, AssertUnwindSafe}; +diff --git a/tests/sync_watch.rs b/tests/sync_watch.rs +index 34f9b78..d4f8ce8 100644 +--- a/tests/sync_watch.rs ++++ b/tests/sync_watch.rs +@@ -213,6 +213,7 @@ fn reopened_after_subscribe() { + } + + #[test] ++#[cfg(panic = "unwind")] + #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding + fn send_modify_panic() { + let (tx, mut rx) = watch::channel("one"); diff --git a/patches/sync_broadcast.patch b/patches/sync_broadcast.patch deleted file mode 100644 index c63a4a7..0000000 --- a/patches/sync_broadcast.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs -index 5f79800..9ef7927 100644 ---- a/tests/sync_broadcast.rs -+++ b/tests/sync_broadcast.rs -@@ -286,6 +286,7 @@ fn capacity_too_big() { - } - - #[test] -+#[cfg(not(target_os = "android"))] - fn panic_in_clone() { - use std::panic::{self, AssertUnwindSafe}; - diff --git a/src/blocking.rs b/src/blocking.rs index f88b1db..f172399 100644 --- a/src/blocking.rs +++ b/src/blocking.rs @@ -1,5 +1,11 @@ cfg_rt! { pub(crate) use crate::runtime::spawn_blocking; + + cfg_fs! { + #[allow(unused_imports)] + pub(crate) use crate::runtime::spawn_mandatory_blocking; + } + pub(crate) use crate::task::JoinHandle; } @@ -16,7 +22,16 @@ cfg_not_rt! { { assert_send_sync::<JoinHandle<std::cell::Cell<()>>>(); panic!("requires the `rt` Tokio feature flag") + } + cfg_fs! { + pub(crate) fn spawn_mandatory_blocking<F, R>(_f: F) -> Option<JoinHandle<R>> + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + panic!("requires the `rt` Tokio feature flag") + } } pub(crate) struct JoinHandle<R> { diff --git a/src/doc/mod.rs b/src/doc/mod.rs index 3a94934..7992fc6 100644 --- a/src/doc/mod.rs +++ b/src/doc/mod.rs @@ -21,4 +21,3 @@ pub enum NotDefinedHere {} pub mod os; -pub mod winapi; diff --git a/src/doc/winapi.rs b/src/doc/winapi.rs deleted file mode 100644 index be68749..0000000 --- a/src/doc/winapi.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! See [winapi]. -//! -//! [winapi]: https://docs.rs/winapi - -/// See [winapi::shared](https://docs.rs/winapi/*/winapi/shared/index.html). -pub mod shared { - /// See [winapi::shared::winerror](https://docs.rs/winapi/*/winapi/shared/winerror/index.html). - #[allow(non_camel_case_types)] - pub mod winerror { - /// See [winapi::shared::winerror::ERROR_ACCESS_DENIED][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_ACCESS_DENIED.html - pub type ERROR_ACCESS_DENIED = crate::doc::NotDefinedHere; - - /// See [winapi::shared::winerror::ERROR_PIPE_BUSY][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_PIPE_BUSY.html - pub type ERROR_PIPE_BUSY = crate::doc::NotDefinedHere; - - /// See [winapi::shared::winerror::ERROR_MORE_DATA][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_MORE_DATA.html - pub type ERROR_MORE_DATA = crate::doc::NotDefinedHere; - } -} - -/// See [winapi::um](https://docs.rs/winapi/*/winapi/um/index.html). -pub mod um { - /// See [winapi::um::winbase](https://docs.rs/winapi/*/winapi/um/winbase/index.html). - #[allow(non_camel_case_types)] - pub mod winbase { - /// See [winapi::um::winbase::PIPE_TYPE_MESSAGE][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_MESSAGE.html - pub type PIPE_TYPE_MESSAGE = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::PIPE_TYPE_BYTE][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_BYTE.html - pub type PIPE_TYPE_BYTE = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::PIPE_CLIENT_END][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_CLIENT_END.html - pub type PIPE_CLIENT_END = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::PIPE_SERVER_END][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_SERVER_END.html - pub type PIPE_SERVER_END = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::SECURITY_IDENTIFICATION][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.SECURITY_IDENTIFICATION.html - pub type SECURITY_IDENTIFICATION = crate::doc::NotDefinedHere; - } - - /// See [winapi::um::minwinbase](https://docs.rs/winapi/*/winapi/um/minwinbase/index.html). - #[allow(non_camel_case_types)] - pub mod minwinbase { - /// See [winapi::um::minwinbase::SECURITY_ATTRIBUTES][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/minwinbase/constant.SECURITY_ATTRIBUTES.html - pub type SECURITY_ATTRIBUTES = crate::doc::NotDefinedHere; - } -} diff --git a/src/fs/file.rs b/src/fs/file.rs index 61071cf..d513beb 100644 --- a/src/fs/file.rs +++ b/src/fs/file.rs @@ -20,16 +20,16 @@ use std::task::Poll; use std::task::Poll::*; #[cfg(test)] -use super::mocks::spawn_blocking; -#[cfg(test)] use super::mocks::JoinHandle; #[cfg(test)] use super::mocks::MockFile as StdFile; -#[cfg(not(test))] -use crate::blocking::spawn_blocking; +#[cfg(test)] +use super::mocks::{spawn_blocking, spawn_mandatory_blocking}; #[cfg(not(test))] use crate::blocking::JoinHandle; #[cfg(not(test))] +use crate::blocking::{spawn_blocking, spawn_mandatory_blocking}; +#[cfg(not(test))] use std::fs::File as StdFile; /// A reference to an open file on the filesystem. @@ -565,29 +565,30 @@ impl AsyncSeek for File { let me = self.get_mut(); let inner = me.inner.get_mut(); - loop { - match inner.state { - Busy(_) => panic!("must wait for poll_complete before calling start_seek"), - Idle(ref mut buf_cell) => { - let mut buf = buf_cell.take().unwrap(); + match inner.state { + Busy(_) => Err(io::Error::new( + io::ErrorKind::Other, + "other file operation is pending, call poll_complete before start_seek", + )), + Idle(ref mut buf_cell) => { + let mut buf = buf_cell.take().unwrap(); - // Factor in any unread data from the buf - if !buf.is_empty() { - let n = buf.discard_read(); + // Factor in any unread data from the buf + if !buf.is_empty() { + let n = buf.discard_read(); - if let SeekFrom::Current(ref mut offset) = pos { - *offset += n; - } + if let SeekFrom::Current(ref mut offset) = pos { + *offset += n; } + } - let std = me.std.clone(); + let std = me.std.clone(); - inner.state = Busy(spawn_blocking(move || { - let res = (&*std).seek(pos); - (Operation::Seek(res), buf) - })); - return Ok(()); - } + inner.state = Busy(spawn_blocking(move || { + let res = (&*std).seek(pos); + (Operation::Seek(res), buf) + })); + Ok(()) } } } @@ -649,7 +650,7 @@ impl AsyncWrite for File { let n = buf.copy_from(src); let std = me.std.clone(); - inner.state = Busy(spawn_blocking(move || { + let blocking_task_join_handle = spawn_mandatory_blocking(move || { let res = if let Some(seek) = seek { (&*std).seek(seek).and_then(|_| buf.write_to(&mut &*std)) } else { @@ -657,7 +658,12 @@ impl AsyncWrite for File { }; (Operation::Write(res), buf) - })); + }) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::Other, "background task failed") + })?; + + inner.state = Busy(blocking_task_join_handle); return Ready(Ok(n)); } diff --git a/src/fs/file/tests.rs b/src/fs/file/tests.rs index 28b5ffe..7c61b3c 100644 --- a/src/fs/file/tests.rs +++ b/src/fs/file/tests.rs @@ -228,14 +228,15 @@ fn flush_while_idle() { } #[test] +#[cfg_attr(miri, ignore)] // takes a really long time with miri fn read_with_buffer_larger_than_max() { // Chunks - let chunk_a = 16 * 1024; + let chunk_a = crate::io::blocking::MAX_BUF; let chunk_b = chunk_a * 2; let chunk_c = chunk_a * 3; let chunk_d = chunk_a * 4; - assert_eq!(chunk_d / 1024, 64); + assert_eq!(chunk_d / 1024 / 1024, 8); let mut data = vec![]; for i in 0..(chunk_d - 1) { @@ -299,14 +300,15 @@ fn read_with_buffer_larger_than_max() { } #[test] +#[cfg_attr(miri, ignore)] // takes a really long time with miri fn write_with_buffer_larger_than_max() { // Chunks - let chunk_a = 16 * 1024; + let chunk_a = crate::io::blocking::MAX_BUF; let chunk_b = chunk_a * 2; let chunk_c = chunk_a * 3; let chunk_d = chunk_a * 4; - assert_eq!(chunk_d / 1024, 64); + assert_eq!(chunk_d / 1024 / 1024, 8); let mut data = vec![]; for i in 0..(chunk_d - 1) { @@ -953,3 +955,24 @@ fn partial_read_set_len_ok() { assert_eq!(n, FOO.len()); assert_eq!(&buf[..n], FOO); } + +#[test] +fn busy_file_seek_error() { + let mut file = MockFile::default(); + let mut seq = Sequence::new(); + file.expect_inner_write() + .once() + .in_sequence(&mut seq) + .returning(|_| Err(io::ErrorKind::Other.into())); + + let mut file = crate::io::BufReader::new(File::from_std(file)); + { + let mut t = task::spawn(file.write(HELLO)); + assert_ready_ok!(t.poll()); + } + + pool::run_one(); + + let mut t = task::spawn(file.seek(SeekFrom::Start(0))); + assert_ready_err!(t.poll()); +} diff --git a/src/fs/mocks.rs b/src/fs/mocks.rs index 68ef4f3..aa01e24 100644 --- a/src/fs/mocks.rs +++ b/src/fs/mocks.rs @@ -81,7 +81,7 @@ impl Write for &'_ MockFile { } } -thread_local! { +tokio_thread_local! { static QUEUE: RefCell<VecDeque<Box<dyn FnOnce() + Send>>> = RefCell::new(VecDeque::new()) } @@ -105,6 +105,21 @@ where JoinHandle { rx } } +pub(super) fn spawn_mandatory_blocking<F, R>(f: F) -> Option<JoinHandle<R>> +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + let (tx, rx) = oneshot::channel(); + let task = Box::new(move || { + let _ = tx.send(f()); + }); + + QUEUE.with(|cell| cell.borrow_mut().push_back(task)); + + Some(JoinHandle { rx }) +} + impl<T> Future for JoinHandle<T> { type Output = Result<T, io::Error>; diff --git a/src/fs/mod.rs b/src/fs/mod.rs index ca0264b..3afefc6 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -22,6 +22,24 @@ //! `std::io::ErrorKind::WouldBlock` if a *worker* thread can not be converted //! to a *backup* thread immediately. //! +//! **Warning**: These adapters may create a large number of temporary tasks, +//! especially when reading large files. When performing a lot of operations +//! in one batch, it may be significantly faster to use [`spawn_blocking`] +//! directly: +//! +//! ``` +//! use tokio::fs::File; +//! use std::io::{BufReader, BufRead}; +//! async fn count_lines(file: File) -> Result<usize, std::io::Error> { +//! let file = file.into_std().await; +//! tokio::task::spawn_blocking(move || { +//! let line_count = BufReader::new(file).lines().count(); +//! Ok(line_count) +//! }).await? +//! } +//! ``` +//! +//! [`spawn_blocking`]: fn@crate::task::spawn_blocking //! [`AsyncRead`]: trait@crate::io::AsyncRead mod canonicalize; diff --git a/src/fs/open_options.rs b/src/fs/open_options.rs index f3b4654..c600ef5 100644 --- a/src/fs/open_options.rs +++ b/src/fs/open_options.rs @@ -542,7 +542,7 @@ feature! { /// # Examples /// /// ```no_run - /// use winapi::um::winbase::FILE_FLAG_DELETE_ON_CLOSE; + /// use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_DELETE_ON_CLOSE; /// use tokio::fs::OpenOptions; /// /// # #[tokio::main] @@ -581,7 +581,7 @@ feature! { /// # Examples /// /// ```no_run - /// use winapi::um::winnt::FILE_ATTRIBUTE_HIDDEN; + /// use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_HIDDEN; /// use tokio::fs::OpenOptions; /// /// # #[tokio::main] @@ -624,7 +624,7 @@ feature! { /// # Examples /// /// ```no_run - /// use winapi::um::winbase::SECURITY_IDENTIFICATION; + /// use windows_sys::Win32::Storage::FileSystem::SECURITY_IDENTIFICATION; /// use tokio::fs::OpenOptions; /// /// # #[tokio::main] diff --git a/src/fs/open_options/mock_open_options.rs b/src/fs/open_options/mock_open_options.rs index cbbda0e..17b4a48 100644 --- a/src/fs/open_options/mock_open_options.rs +++ b/src/fs/open_options/mock_open_options.rs @@ -1,3 +1,4 @@ +#![allow(unreachable_pub)] //! Mock version of std::fs::OpenOptions; use mockall::mock; diff --git a/src/fs/read_dir.rs b/src/fs/read_dir.rs index 281ea4c..9471e8c 100644 --- a/src/fs/read_dir.rs +++ b/src/fs/read_dir.rs @@ -1,5 +1,6 @@ use crate::fs::asyncify; +use std::collections::VecDeque; use std::ffi::OsString; use std::fs::{FileType, Metadata}; use std::future::Future; @@ -19,6 +20,8 @@ use crate::blocking::spawn_blocking; #[cfg(not(test))] use crate::blocking::JoinHandle; +const CHUNK_SIZE: usize = 32; + /// Returns a stream over the entries within a directory. /// /// This is an async version of [`std::fs::read_dir`](std::fs::read_dir) @@ -29,12 +32,17 @@ use crate::blocking::JoinHandle; /// [`spawn_blocking`]: crate::task::spawn_blocking pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> { let path = path.as_ref().to_owned(); - let std = asyncify(|| std::fs::read_dir(path)).await?; + asyncify(|| -> io::Result<ReadDir> { + let mut std = std::fs::read_dir(path)?; + let mut buf = VecDeque::with_capacity(CHUNK_SIZE); + ReadDir::next_chunk(&mut buf, &mut std); - Ok(ReadDir(State::Idle(Some(std)))) + Ok(ReadDir(State::Idle(Some((buf, std))))) + }) + .await } -/// Reads the the entries in a directory. +/// Reads the entries in a directory. /// /// This struct is returned from the [`read_dir`] function of this module and /// will yield instances of [`DirEntry`]. Through a [`DirEntry`] information @@ -58,8 +66,8 @@ pub struct ReadDir(State); #[derive(Debug)] enum State { - Idle(Option<std::fs::ReadDir>), - Pending(JoinHandle<(Option<io::Result<std::fs::DirEntry>>, std::fs::ReadDir)>), + Idle(Option<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir)>), + Pending(JoinHandle<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir)>), } impl ReadDir { @@ -94,29 +102,57 @@ impl ReadDir { pub fn poll_next_entry(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<DirEntry>>> { loop { match self.0 { - State::Idle(ref mut std) => { - let mut std = std.take().unwrap(); + State::Idle(ref mut data) => { + let (buf, _) = data.as_mut().unwrap(); + + if let Some(ent) = buf.pop_front() { + return Poll::Ready(ent.map(Some)); + }; + + let (mut buf, mut std) = data.take().unwrap(); self.0 = State::Pending(spawn_blocking(move || { - let ret = std.next(); - (ret, std) + ReadDir::next_chunk(&mut buf, &mut std); + (buf, std) })); } State::Pending(ref mut rx) => { - let (ret, std) = ready!(Pin::new(rx).poll(cx))?; - self.0 = State::Idle(Some(std)); + let (mut buf, std) = ready!(Pin::new(rx).poll(cx))?; - let ret = match ret { - Some(Ok(std)) => Ok(Some(DirEntry(Arc::new(std)))), + let ret = match buf.pop_front() { + Some(Ok(x)) => Ok(Some(x)), Some(Err(e)) => Err(e), None => Ok(None), }; + self.0 = State::Idle(Some((buf, std))); + return Poll::Ready(ret); } } } } + + fn next_chunk(buf: &mut VecDeque<io::Result<DirEntry>>, std: &mut std::fs::ReadDir) { + for ret in std.by_ref().take(CHUNK_SIZE) { + let success = ret.is_ok(); + + buf.push_back(ret.map(|std| DirEntry { + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks" + )))] + file_type: std.file_type().ok(), + std: Arc::new(std), + })); + + if !success { + break; + } + } + } } feature! { @@ -160,7 +196,16 @@ feature! { /// filesystem. Each entry can be inspected via methods to learn about the full /// path or possibly other metadata through per-platform extension traits. #[derive(Debug)] -pub struct DirEntry(Arc<std::fs::DirEntry>); +pub struct DirEntry { + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks" + )))] + file_type: Option<FileType>, + std: Arc<std::fs::DirEntry>, +} impl DirEntry { /// Returns the full path to the file that this entry represents. @@ -193,7 +238,7 @@ impl DirEntry { /// /// The exact text, of course, depends on what files you have in `.`. pub fn path(&self) -> PathBuf { - self.0.path() + self.std.path() } /// Returns the bare file name of this directory entry without any other @@ -214,7 +259,7 @@ impl DirEntry { /// # } /// ``` pub fn file_name(&self) -> OsString { - self.0.file_name() + self.std.file_name() } /// Returns the metadata for the file that this entry points at. @@ -248,7 +293,7 @@ impl DirEntry { /// # } /// ``` pub async fn metadata(&self) -> io::Result<Metadata> { - let std = self.0.clone(); + let std = self.std.clone(); asyncify(move || std.metadata()).await } @@ -283,13 +328,23 @@ impl DirEntry { /// # } /// ``` pub async fn file_type(&self) -> io::Result<FileType> { - let std = self.0.clone(); + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks" + )))] + if let Some(file_type) = self.file_type { + return Ok(file_type); + } + + let std = self.std.clone(); asyncify(move || std.file_type()).await } /// Returns a reference to the underlying `std::fs::DirEntry`. #[cfg(unix)] pub(super) fn as_inner(&self) -> &std::fs::DirEntry { - &self.0 + &self.std } } diff --git a/src/future/block_on.rs b/src/future/block_on.rs index 91f9cc0..2c2ab37 100644 --- a/src/future/block_on.rs +++ b/src/future/block_on.rs @@ -1,15 +1,22 @@ use std::future::Future; cfg_rt! { + #[track_caller] pub(crate) fn block_on<F: Future>(f: F) -> F::Output { - let mut e = crate::runtime::enter::enter(false); + let mut e = crate::runtime::context::try_enter_blocking_region().expect( + "Cannot block the current thread from within a runtime. This \ + happens because a function attempted to block the current \ + thread while the thread is being used to drive asynchronous \ + tasks." + ); e.block_on(f).unwrap() } } cfg_not_rt! { + #[track_caller] pub(crate) fn block_on<F: Future>(f: F) -> F::Output { - let mut park = crate::park::thread::CachedParkThread::new(); + let mut park = crate::runtime::park::CachedParkThread::new(); park.block_on(f).unwrap() } } diff --git a/src/future/mod.rs b/src/future/mod.rs index 96483ac..084ddc5 100644 --- a/src/future/mod.rs +++ b/src/future/mod.rs @@ -8,11 +8,6 @@ pub(crate) mod maybe_done; mod poll_fn; pub use poll_fn::poll_fn; -cfg_not_loom! { - mod ready; - pub(crate) use ready::{ok, Ready}; -} - cfg_process! { mod try_join; pub(crate) use try_join::try_join3; diff --git a/src/future/poll_fn.rs b/src/future/poll_fn.rs index d82ce89..074d943 100644 --- a/src/future/poll_fn.rs +++ b/src/future/poll_fn.rs @@ -7,13 +7,23 @@ use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; +// This struct is intentionally `!Unpin` when `F` is `!Unpin`. This is to +// mitigate the issue where rust puts noalias on mutable references to the +// `PollFn` type if it is `Unpin`. If the closure has ownership of a future, +// then this "leaks" and the future is affected by noalias too, which we don't +// want. +// +// See this thread for more information: +// <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484> +// +// The fact that `PollFn` is not `Unpin` when it shouldn't be is tested in +// `tests/async_send_sync.rs`. + /// Future for the [`poll_fn`] function. pub struct PollFn<F> { f: F, } -impl<F> Unpin for PollFn<F> {} - /// Creates a new future wrapping around a function returning [`Poll`]. pub fn poll_fn<T, F>(f: F) -> PollFn<F> where @@ -34,7 +44,17 @@ where { type Output = T; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<T> { - (&mut self.f)(cx) + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<T> { + // Safety: We never construct a `Pin<&mut F>` anywhere, so accessing `f` + // mutably in an unpinned way is sound. + // + // This use of unsafe cannot be replaced with the pin-project macro + // because: + // * If we put `#[pin]` on the field, then it gives us a `Pin<&mut F>`, + // which we can't use to call the closure. + // * If we don't put `#[pin]` on the field, then it makes `PollFn` be + // unconditionally `Unpin`, which we also don't want. + let me = unsafe { Pin::into_inner_unchecked(self) }; + (me.f)(cx) } } diff --git a/src/future/ready.rs b/src/future/ready.rs deleted file mode 100644 index de2d60c..0000000 --- a/src/future/ready.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::future::Future; -use std::pin::Pin; -use std::task::{Context, Poll}; - -/// Future for the [`ok`](ok()) function. -/// -/// `pub` in order to use the future as an associated type in a sealed trait. -#[derive(Debug)] -// Used as an associated type in a "sealed" trait. -#[allow(unreachable_pub)] -pub struct Ready<T>(Option<T>); - -impl<T> Unpin for Ready<T> {} - -impl<T> Future for Ready<T> { - type Output = T; - - #[inline] - fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<T> { - Poll::Ready(self.0.take().unwrap()) - } -} - -/// Creates a future that is immediately ready with a success value. -pub(crate) fn ok<T, E>(t: T) -> Ready<Result<T, E>> { - Ready(Some(Ok(t))) -} diff --git a/src/io/async_fd.rs b/src/io/async_fd.rs index 9ec5b7f..92fc6b3 100644 --- a/src/io/async_fd.rs +++ b/src/io/async_fd.rs @@ -1,4 +1,6 @@ -use crate::io::driver::{Handle, Interest, ReadyEvent, Registration}; +use crate::io::Interest; +use crate::runtime::io::{ReadyEvent, Registration}; +use crate::runtime::scheduler; use mio::unix::SourceFd; use std::io; @@ -81,6 +83,7 @@ use std::{task::Context, task::Poll}; /// /// impl AsyncTcpStream { /// pub fn new(tcp: TcpStream) -> io::Result<Self> { +/// tcp.set_nonblocking(true)?; /// Ok(Self { /// inner: AsyncFd::new(tcp)?, /// }) @@ -166,12 +169,18 @@ pub struct AsyncFdReadyMutGuard<'a, T: AsRawFd> { const ALL_INTEREST: Interest = Interest::READABLE.add(Interest::WRITABLE); impl<T: AsRawFd> AsyncFd<T> { - #[inline] /// Creates an AsyncFd backed by (and taking ownership of) an object /// implementing [`AsRawFd`]. The backing file descriptor is cached at the /// time of creation. /// /// This method must be called in the context of a tokio runtime. + /// + /// # Panics + /// + /// This function panics if there is no current reactor set, or if the `rt` + /// feature flag is not enabled. + #[inline] + #[track_caller] pub fn new(inner: T) -> io::Result<Self> where T: AsRawFd, @@ -179,19 +188,26 @@ impl<T: AsRawFd> AsyncFd<T> { Self::with_interest(inner, ALL_INTEREST) } - #[inline] /// Creates new instance as `new` with additional ability to customize interest, /// allowing to specify whether file descriptor will be polled for read, write or both. + /// + /// # Panics + /// + /// This function panics if there is no current reactor set, or if the `rt` + /// feature flag is not enabled. + #[inline] + #[track_caller] pub fn with_interest(inner: T, interest: Interest) -> io::Result<Self> where T: AsRawFd, { - Self::new_with_handle_and_interest(inner, Handle::current(), interest) + Self::new_with_handle_and_interest(inner, scheduler::Handle::current(), interest) } + #[track_caller] pub(crate) fn new_with_handle_and_interest( inner: T, - handle: Handle, + handle: scheduler::Handle, interest: Interest, ) -> io::Result<Self> { let fd = inner.as_raw_fd(); @@ -525,7 +541,7 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> { #[cfg_attr(docsrs, doc(alias = "with_io"))] pub fn try_io<R>( &mut self, - f: impl FnOnce(&AsyncFd<Inner>) -> io::Result<R>, + f: impl FnOnce(&'a AsyncFd<Inner>) -> io::Result<R>, ) -> Result<io::Result<R>, TryIoError> { let result = f(self.async_fd); @@ -542,12 +558,12 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> { } /// Returns a shared reference to the inner [`AsyncFd`]. - pub fn get_ref(&self) -> &AsyncFd<Inner> { + pub fn get_ref(&self) -> &'a AsyncFd<Inner> { self.async_fd } /// Returns a shared reference to the backing object of the inner [`AsyncFd`]. - pub fn get_inner(&self) -> &Inner { + pub fn get_inner(&self) -> &'a Inner { self.get_ref().get_ref() } } @@ -598,7 +614,7 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyMutGuard<'a, Inner> { &mut self, f: impl FnOnce(&mut AsyncFd<Inner>) -> io::Result<R>, ) -> Result<io::Result<R>, TryIoError> { - let result = f(&mut self.async_fd); + let result = f(self.async_fd); if let Err(e) = result.as_ref() { if e.kind() == io::ErrorKind::WouldBlock { diff --git a/src/io/blocking.rs b/src/io/blocking.rs index 1d79ee7..416573e 100644 --- a/src/io/blocking.rs +++ b/src/io/blocking.rs @@ -26,7 +26,7 @@ pub(crate) struct Buf { pos: usize, } -pub(crate) const MAX_BUF: usize = 16 * 1024; +pub(crate) const MAX_BUF: usize = 2 * 1024 * 1024; #[derive(Debug)] enum State<T> { @@ -34,8 +34,9 @@ enum State<T> { Busy(sys::Blocking<(io::Result<usize>, Buf, T)>), } -cfg_io_std! { +cfg_io_blocking! { impl<T> Blocking<T> { + #[cfg_attr(feature = "fs", allow(dead_code))] pub(crate) fn new(inner: T) -> Blocking<T> { Blocking { inner: Some(inner), diff --git a/src/io/bsd/poll_aio.rs b/src/io/bsd/poll_aio.rs index f1ac4b2..6ac9e28 100644 --- a/src/io/bsd/poll_aio.rs +++ b/src/io/bsd/poll_aio.rs @@ -1,6 +1,8 @@ //! Use POSIX AIO futures with Tokio. -use crate::io::driver::{Handle, Interest, ReadyEvent, Registration}; +use crate::io::interest::Interest; +use crate::runtime::io::{ReadyEvent, Registration}; +use crate::runtime::scheduler; use mio::event::Source; use mio::Registry; use mio::Token; @@ -117,7 +119,7 @@ impl<E: AioSource> Aio<E> { fn new_with_interest(io: E, interest: Interest) -> io::Result<Self> { let mut io = MioSource(io); - let handle = Handle::current(); + let handle = scheduler::Handle::current(); let registration = Registration::new_with_interest_and_handle(&mut io, interest, handle)?; Ok(Self { io, registration }) } diff --git a/src/io/driver/interest.rs b/src/io/interest.rs index d6b46df..013c114 100644 --- a/src/io/driver/interest.rs +++ b/src/io/interest.rs @@ -1,6 +1,6 @@ #![cfg_attr(not(feature = "net"), allow(dead_code, unreachable_pub))] -use crate::io::driver::Ready; +use crate::io::ready::Ready; use std::fmt; use std::ops; @@ -100,7 +100,7 @@ impl Interest { self.0 } - pub(super) fn mask(self) -> Ready { + pub(crate) fn mask(self) -> Ready { match self { Interest::READABLE => Ready::READABLE | Ready::READ_CLOSED, Interest::WRITABLE => Ready::WRITABLE | Ready::WRITE_CLOSED, diff --git a/src/io/mod.rs b/src/io/mod.rs index cfdda61..f48035a 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -1,5 +1,3 @@ -#![cfg_attr(loom, allow(dead_code, unreachable_pub))] - //! Traits, helpers, and type definitions for asynchronous I/O functionality. //! //! This module is the asynchronous version of `std::io`. Primarily, it @@ -180,6 +178,12 @@ //! [`Sink`]: https://docs.rs/futures/0.3/futures/sink/trait.Sink.html //! [`Stream`]: https://docs.rs/futures/0.3/futures/stream/trait.Stream.html //! [`Write`]: std::io::Write + +#![cfg_attr( + not(all(feature = "rt", feature = "net")), + allow(dead_code, unused_imports) +)] + cfg_io_blocking! { pub(crate) mod blocking; } @@ -205,15 +209,19 @@ pub use self::read_buf::ReadBuf; pub use std::io::{Error, ErrorKind, Result, SeekFrom}; cfg_io_driver_impl! { - pub(crate) mod driver; + pub(crate) mod interest; + pub(crate) mod ready; cfg_net! { - pub use driver::{Interest, Ready}; + pub use interest::Interest; + pub use ready::Ready; } + #[cfg_attr(tokio_wasi, allow(unused_imports))] mod poll_evented; #[cfg(not(loom))] + #[cfg_attr(tokio_wasi, allow(unused_imports))] pub(crate) use poll_evented::PollEvented; } diff --git a/src/io/poll_evented.rs b/src/io/poll_evented.rs index 44e68a2..dfe9ae3 100644 --- a/src/io/poll_evented.rs +++ b/src/io/poll_evented.rs @@ -1,16 +1,19 @@ -use crate::io::driver::{Handle, Interest, Registration}; +use crate::io::interest::Interest; +use crate::runtime::io::Registration; +use crate::runtime::scheduler; use mio::event::Source; use std::fmt; use std::io; use std::ops::Deref; +use std::panic::{RefUnwindSafe, UnwindSafe}; cfg_io_driver! { /// Associates an I/O resource that implements the [`std::io::Read`] and/or /// [`std::io::Write`] traits with the reactor that drives it. /// /// `PollEvented` uses [`Registration`] internally to take a type that - /// implements [`mio::event::Source`] as well as [`std::io::Read`] and or + /// implements [`mio::event::Source`] as well as [`std::io::Read`] and/or /// [`std::io::Write`] and associate it with a reactor that will drive it. /// /// Once the [`mio::event::Source`] type is wrapped by `PollEvented`, it can be @@ -40,12 +43,12 @@ cfg_io_driver! { /// [`poll_read_ready`] again will also indicate read readiness. /// /// When the operation is attempted and is unable to succeed due to the I/O - /// resource not being ready, the caller must call `clear_readiness`. + /// resource not being ready, the caller must call [`clear_readiness`]. /// This clears the readiness state until a new readiness event is received. /// /// This allows the caller to implement additional functions. For example, /// [`TcpListener`] implements poll_accept by using [`poll_read_ready`] and - /// `clear_read_ready`. + /// [`clear_readiness`]. /// /// ## Platform-specific events /// @@ -56,6 +59,7 @@ cfg_io_driver! { /// [`AsyncRead`]: crate::io::AsyncRead /// [`AsyncWrite`]: crate::io::AsyncWrite /// [`TcpListener`]: crate::net::TcpListener + /// [`clear_readiness`]: Registration::clear_readiness /// [`poll_read_ready`]: Registration::poll_read_ready /// [`poll_write_ready`]: Registration::poll_write_ready pub(crate) struct PollEvented<E: Source> { @@ -76,6 +80,7 @@ impl<E: Source> PollEvented<E> { /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] #[cfg_attr(feature = "signal", allow(unused))] pub(crate) fn new(io: E) -> io::Result<Self> { PollEvented::new_with_interest(io, Interest::READABLE | Interest::WRITABLE) @@ -96,15 +101,17 @@ impl<E: Source> PollEvented<E> { /// a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) /// function. + #[track_caller] #[cfg_attr(feature = "signal", allow(unused))] pub(crate) fn new_with_interest(io: E, interest: Interest) -> io::Result<Self> { - Self::new_with_interest_and_handle(io, interest, Handle::current()) + Self::new_with_interest_and_handle(io, interest, scheduler::Handle::current()) } + #[track_caller] pub(crate) fn new_with_interest_and_handle( mut io: E, interest: Interest, - handle: Handle, + handle: scheduler::Handle, ) -> io::Result<Self> { let registration = Registration::new_with_interest_and_handle(&mut io, interest, handle)?; Ok(Self { @@ -114,11 +121,7 @@ impl<E: Source> PollEvented<E> { } /// Returns a reference to the registration. - #[cfg(any( - feature = "net", - all(unix, feature = "process"), - all(unix, feature = "signal"), - ))] + #[cfg(any(feature = "net"))] pub(crate) fn registration(&self) -> &Registration { &self.registration } @@ -133,7 +136,7 @@ impl<E: Source> PollEvented<E> { } feature! { - #![any(feature = "net", feature = "process")] + #![any(feature = "net", all(unix, feature = "process"))] use crate::io::ReadBuf; use std::task::{Context, Poll}; @@ -150,16 +153,32 @@ feature! { { use std::io::Read; - let n = ready!(self.registration.poll_read_io(cx, || { + loop { + let evt = ready!(self.registration.poll_read_ready(cx))?; + let b = &mut *(buf.unfilled_mut() as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]); - self.io.as_ref().unwrap().read(b) - }))?; - - // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the - // buffer. - buf.assume_init(n); - buf.advance(n); - Poll::Ready(Ok(())) + let len = b.len(); + + match self.io.as_ref().unwrap().read(b) { + Ok(n) => { + // if we read a partially full buffer, this is sufficient on unix to show + // that the socket buffer has been drained + if n > 0 && (!cfg!(windows) && n < len) { + self.registration.clear_readiness(evt); + } + + // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the + // buffer. + buf.assume_init(n); + buf.advance(n); + return Poll::Ready(Ok(())); + }, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.registration.clear_readiness(evt); + } + Err(e) => return Poll::Ready(Err(e)), + } + } } pub(crate) fn poll_write<'a>(&'a self, cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> @@ -167,10 +186,29 @@ feature! { &'a E: io::Write + 'a, { use std::io::Write; - self.registration.poll_write_io(cx, || self.io.as_ref().unwrap().write(buf)) + + loop { + let evt = ready!(self.registration.poll_write_ready(cx))?; + + match self.io.as_ref().unwrap().write(buf) { + Ok(n) => { + // if we write only part of our buffer, this is sufficient on unix to show + // that the socket buffer is full + if n > 0 && (!cfg!(windows) && n < buf.len()) { + self.registration.clear_readiness(evt); + } + + return Poll::Ready(Ok(n)); + }, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.registration.clear_readiness(evt); + } + Err(e) => return Poll::Ready(Err(e)), + } + } } - #[cfg(feature = "net")] + #[cfg(any(feature = "net", feature = "process"))] pub(crate) fn poll_write_vectored<'a>( &'a self, cx: &mut Context<'_>, @@ -185,6 +223,10 @@ feature! { } } +impl<E: Source> UnwindSafe for PollEvented<E> {} + +impl<E: Source> RefUnwindSafe for PollEvented<E> {} + impl<E: Source> Deref for PollEvented<E> { type Target = E; diff --git a/src/io/read_buf.rs b/src/io/read_buf.rs index ad58cbe..0dc595a 100644 --- a/src/io/read_buf.rs +++ b/src/io/read_buf.rs @@ -1,9 +1,5 @@ -// This lint claims ugly casting is somehow safer than transmute, but there's -// no evidence that is the case. Shush. -#![allow(clippy::transmute_ptr_to_ptr)] - use std::fmt; -use std::mem::{self, MaybeUninit}; +use std::mem::MaybeUninit; /// A wrapper around a byte buffer that is incrementally filled and initialized. /// @@ -35,7 +31,7 @@ impl<'a> ReadBuf<'a> { #[inline] pub fn new(buf: &'a mut [u8]) -> ReadBuf<'a> { let initialized = buf.len(); - let buf = unsafe { mem::transmute::<&mut [u8], &mut [MaybeUninit<u8>]>(buf) }; + let buf = unsafe { slice_to_uninit_mut(buf) }; ReadBuf { buf, filled: 0, @@ -67,8 +63,7 @@ impl<'a> ReadBuf<'a> { let slice = &self.buf[..self.filled]; // safety: filled describes how far into the buffer that the // user has filled with bytes, so it's been initialized. - // TODO: This could use `MaybeUninit::slice_get_ref` when it is stable. - unsafe { mem::transmute::<&[MaybeUninit<u8>], &[u8]>(slice) } + unsafe { slice_assume_init(slice) } } /// Returns a mutable reference to the filled portion of the buffer. @@ -77,8 +72,7 @@ impl<'a> ReadBuf<'a> { let slice = &mut self.buf[..self.filled]; // safety: filled describes how far into the buffer that the // user has filled with bytes, so it's been initialized. - // TODO: This could use `MaybeUninit::slice_get_mut` when it is stable. - unsafe { mem::transmute::<&mut [MaybeUninit<u8>], &mut [u8]>(slice) } + unsafe { slice_assume_init_mut(slice) } } /// Returns a new `ReadBuf` comprised of the unfilled section up to `n`. @@ -97,8 +91,7 @@ impl<'a> ReadBuf<'a> { let slice = &self.buf[..self.initialized]; // safety: initialized describes how far into the buffer that the // user has at some point initialized with bytes. - // TODO: This could use `MaybeUninit::slice_get_ref` when it is stable. - unsafe { mem::transmute::<&[MaybeUninit<u8>], &[u8]>(slice) } + unsafe { slice_assume_init(slice) } } /// Returns a mutable reference to the initialized portion of the buffer. @@ -109,15 +102,14 @@ impl<'a> ReadBuf<'a> { let slice = &mut self.buf[..self.initialized]; // safety: initialized describes how far into the buffer that the // user has at some point initialized with bytes. - // TODO: This could use `MaybeUninit::slice_get_mut` when it is stable. - unsafe { mem::transmute::<&mut [MaybeUninit<u8>], &mut [u8]>(slice) } + unsafe { slice_assume_init_mut(slice) } } /// Returns a mutable reference to the entire buffer, without ensuring that it has been fully /// initialized. /// /// The elements between 0 and `self.filled().len()` are filled, and those between 0 and - /// `self.initialized().len()` are initialized (and so can be transmuted to a `&mut [u8]`). + /// `self.initialized().len()` are initialized (and so can be converted to a `&mut [u8]`). /// /// The caller of this method must ensure that these invariants are upheld. For example, if the /// caller initializes some of the uninitialized section of the buffer, it must call @@ -160,6 +152,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if `self.remaining()` is less than `n`. #[inline] + #[track_caller] pub fn initialize_unfilled_to(&mut self, n: usize) -> &mut [u8] { assert!(self.remaining() >= n, "n overflows remaining"); @@ -178,7 +171,7 @@ impl<'a> ReadBuf<'a> { let slice = &mut self.buf[self.filled..end]; // safety: just above, we checked that the end of the buf has // been initialized to some value. - unsafe { mem::transmute::<&mut [MaybeUninit<u8>], &mut [u8]>(slice) } + unsafe { slice_assume_init_mut(slice) } } /// Returns the number of bytes at the end of the slice that have not yet been filled. @@ -203,6 +196,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if the filled region of the buffer would become larger than the initialized region. #[inline] + #[track_caller] pub fn advance(&mut self, n: usize) { let new = self.filled.checked_add(n).expect("filled overflow"); self.set_filled(new); @@ -219,6 +213,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if the filled region of the buffer would become larger than the initialized region. #[inline] + #[track_caller] pub fn set_filled(&mut self, n: usize) { assert!( n <= self.initialized, @@ -249,6 +244,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if `self.remaining()` is less than `buf.len()`. #[inline] + #[track_caller] pub fn put_slice(&mut self, buf: &[u8]) { assert!( self.remaining() >= buf.len(), @@ -283,3 +279,17 @@ impl fmt::Debug for ReadBuf<'_> { .finish() } } + +unsafe fn slice_to_uninit_mut(slice: &mut [u8]) -> &mut [MaybeUninit<u8>] { + &mut *(slice as *mut [u8] as *mut [MaybeUninit<u8>]) +} + +// TODO: This could use `MaybeUninit::slice_assume_init` when it is stable. +unsafe fn slice_assume_init(slice: &[MaybeUninit<u8>]) -> &[u8] { + &*(slice as *const [MaybeUninit<u8>] as *const [u8]) +} + +// TODO: This could use `MaybeUninit::slice_assume_init_mut` when it is stable. +unsafe fn slice_assume_init_mut(slice: &mut [MaybeUninit<u8>]) -> &mut [u8] { + &mut *(slice as *mut [MaybeUninit<u8>] as *mut [u8]) +} diff --git a/src/io/driver/ready.rs b/src/io/ready.rs index 2430d30..ef135c4 100644 --- a/src/io/driver/ready.rs +++ b/src/io/ready.rs @@ -12,7 +12,7 @@ const WRITE_CLOSED: usize = 0b0_1000; /// /// `Ready` tracks which operation an I/O resource is ready to perform. #[cfg_attr(docsrs, doc(cfg(feature = "net")))] -#[derive(Clone, Copy, PartialEq, PartialOrd)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct Ready(usize); impl Ready { diff --git a/src/io/split.rs b/src/io/split.rs index 8258a0f..f067b65 100644 --- a/src/io/split.rs +++ b/src/io/split.rs @@ -74,7 +74,11 @@ impl<T> ReadHalf<T> { /// same `split` operation this method will panic. /// This can be checked ahead of time by comparing the stream ID /// of the two halves. - pub fn unsplit(self, wr: WriteHalf<T>) -> T { + #[track_caller] + pub fn unsplit(self, wr: WriteHalf<T>) -> T + where + T: Unpin, + { if self.is_pair_of(&wr) { drop(wr); diff --git a/src/io/stdio_common.rs b/src/io/stdio_common.rs index 7e4a198..b1cc61d 100644 --- a/src/io/stdio_common.rs +++ b/src/io/stdio_common.rs @@ -42,7 +42,7 @@ where // for further code. Since `AsyncWrite` can always shrink // buffer at its discretion, excessive (i.e. in tests) shrinking // does not break correctness. - // 2. If buffer is small, it will not be shrinked. + // 2. If buffer is small, it will not be shrunk. // That's why, it's "textness" will not change, so we don't have // to fixup it. if cfg!(not(any(target_os = "windows", test))) || buf.len() <= crate::io::blocking::MAX_BUF @@ -108,14 +108,13 @@ where #[cfg(test)] #[cfg(not(loom))] mod tests { + use crate::io::blocking::MAX_BUF; use crate::io::AsyncWriteExt; use std::io; use std::pin::Pin; use std::task::Context; use std::task::Poll; - const MAX_BUF: usize = 16 * 1024; - struct TextMockWriter; impl crate::io::AsyncWrite for TextMockWriter { @@ -193,7 +192,7 @@ mod tests { fn test_pseudo_text() { // In this test we write a piece of binary data, whose beginning is // text though. We then validate that even in this corner case buffer - // was not shrinked too much. + // was not shrunk too much. let checked_count = super::MAGIC_CONST * super::MAX_BYTES_PER_CHAR; let mut data: Vec<u8> = str::repeat("a", checked_count).into(); data.extend(std::iter::repeat(0b1010_1010).take(MAX_BUF - checked_count + 1)); @@ -212,7 +211,7 @@ mod tests { writer.write_history.iter().copied().sum::<usize>(), data.len() ); - // Check that at most MAX_BYTES_PER_CHAR + 1 (i.e. 5) bytes were shrinked + // Check that at most MAX_BYTES_PER_CHAR + 1 (i.e. 5) bytes were shrunk // from the buffer: one because it was outside of MAX_BUF boundary, and // up to one "utf8 code point". assert!(data.len() - writer.write_history[0] <= super::MAX_BYTES_PER_CHAR + 1); diff --git a/src/io/util/async_seek_ext.rs b/src/io/util/async_seek_ext.rs index 46b3e6c..aadf3a7 100644 --- a/src/io/util/async_seek_ext.rs +++ b/src/io/util/async_seek_ext.rs @@ -69,7 +69,7 @@ cfg_io_util! { /// Creates a future which will rewind to the beginning of the stream. /// - /// This is convenience method, equivalent to to `self.seek(SeekFrom::Start(0))`. + /// This is convenience method, equivalent to `self.seek(SeekFrom::Start(0))`. fn rewind(&mut self) -> Seek<'_, Self> where Self: Unpin, diff --git a/src/io/util/async_write_ext.rs b/src/io/util/async_write_ext.rs index 93a3183..dfdde82 100644 --- a/src/io/util/async_write_ext.rs +++ b/src/io/util/async_write_ext.rs @@ -406,7 +406,7 @@ cfg_io_util! { /// ``` fn write_u8(&mut self, n: u8) -> WriteU8; - /// Writes an unsigned 8-bit integer to the underlying writer. + /// Writes a signed 8-bit integer to the underlying writer. /// /// Equivalent to: /// @@ -425,7 +425,7 @@ cfg_io_util! { /// /// # Examples /// - /// Write unsigned 8 bit integers to a `AsyncWrite`: + /// Write signed 8 bit integers to a `AsyncWrite`: /// /// ```rust /// use tokio::io::{self, AsyncWriteExt}; @@ -434,10 +434,10 @@ cfg_io_util! { /// async fn main() -> io::Result<()> { /// let mut writer = Vec::new(); /// - /// writer.write_u8(2).await?; - /// writer.write_u8(5).await?; + /// writer.write_i8(-2).await?; + /// writer.write_i8(126).await?; /// - /// assert_eq!(writer, b"\x02\x05"); + /// assert_eq!(writer, b"\xFE\x7E"); /// Ok(()) /// } /// ``` diff --git a/src/io/util/buf_reader.rs b/src/io/util/buf_reader.rs index 7df610b..60879c0 100644 --- a/src/io/util/buf_reader.rs +++ b/src/io/util/buf_reader.rs @@ -204,7 +204,6 @@ impl<R: AsyncRead + AsyncSeek> AsyncSeek for BufReader<R> { self.as_mut() .get_pin_mut() .start_seek(SeekFrom::Current(offset))?; - self.as_mut().get_pin_mut().poll_complete(cx)? } else { // seek backwards by our remainder, and then by the offset self.as_mut() @@ -221,8 +220,8 @@ impl<R: AsyncRead + AsyncSeek> AsyncSeek for BufReader<R> { self.as_mut() .get_pin_mut() .start_seek(SeekFrom::Current(n))?; - self.as_mut().get_pin_mut().poll_complete(cx)? } + self.as_mut().get_pin_mut().poll_complete(cx)? } SeekState::PendingOverflowed(n) => { if self.as_mut().get_pin_mut().poll_complete(cx)?.is_pending() { diff --git a/src/io/util/copy.rs b/src/io/util/copy.rs index d0ab7cb..47dad89 100644 --- a/src/io/util/copy.rs +++ b/src/io/util/copy.rs @@ -27,6 +27,51 @@ impl CopyBuffer { } } + fn poll_fill_buf<R>( + &mut self, + cx: &mut Context<'_>, + reader: Pin<&mut R>, + ) -> Poll<io::Result<()>> + where + R: AsyncRead + ?Sized, + { + let me = &mut *self; + let mut buf = ReadBuf::new(&mut me.buf); + buf.set_filled(me.cap); + + let res = reader.poll_read(cx, &mut buf); + if let Poll::Ready(Ok(_)) = res { + let filled_len = buf.filled().len(); + me.read_done = me.cap == filled_len; + me.cap = filled_len; + } + res + } + + fn poll_write_buf<R, W>( + &mut self, + cx: &mut Context<'_>, + mut reader: Pin<&mut R>, + mut writer: Pin<&mut W>, + ) -> Poll<io::Result<usize>> + where + R: AsyncRead + ?Sized, + W: AsyncWrite + ?Sized, + { + let me = &mut *self; + match writer.as_mut().poll_write(cx, &me.buf[me.pos..me.cap]) { + Poll::Pending => { + // Top up the buffer towards full if we can read a bit more + // data - this should improve the chances of a large write + if !me.read_done && me.cap < me.buf.len() { + ready!(me.poll_fill_buf(cx, reader.as_mut()))?; + } + Poll::Pending + } + res => res, + } + } + pub(super) fn poll_copy<R, W>( &mut self, cx: &mut Context<'_>, @@ -41,10 +86,10 @@ impl CopyBuffer { // If our buffer is empty, then we need to read some data to // continue. if self.pos == self.cap && !self.read_done { - let me = &mut *self; - let mut buf = ReadBuf::new(&mut me.buf); + self.pos = 0; + self.cap = 0; - match reader.as_mut().poll_read(cx, &mut buf) { + match self.poll_fill_buf(cx, reader.as_mut()) { Poll::Ready(Ok(_)) => (), Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), Poll::Pending => { @@ -58,20 +103,11 @@ impl CopyBuffer { return Poll::Pending; } } - - let n = buf.filled().len(); - if n == 0 { - self.read_done = true; - } else { - self.pos = 0; - self.cap = n; - } } // If our buffer has some data, let's write it out! while self.pos < self.cap { - let me = &mut *self; - let i = ready!(writer.as_mut().poll_write(cx, &me.buf[me.pos..me.cap]))?; + let i = ready!(self.poll_write_buf(cx, reader.as_mut(), writer.as_mut()))?; if i == 0 { return Poll::Ready(Err(io::Error::new( io::ErrorKind::WriteZero, diff --git a/src/io/util/empty.rs b/src/io/util/empty.rs index f964d18..9e648f8 100644 --- a/src/io/util/empty.rs +++ b/src/io/util/empty.rs @@ -50,16 +50,18 @@ impl AsyncRead for Empty { #[inline] fn poll_read( self: Pin<&mut Self>, - _: &mut Context<'_>, + cx: &mut Context<'_>, _: &mut ReadBuf<'_>, ) -> Poll<io::Result<()>> { + ready!(poll_proceed_and_make_progress(cx)); Poll::Ready(Ok(())) } } impl AsyncBufRead for Empty { #[inline] - fn poll_fill_buf(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<&[u8]>> { + fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> { + ready!(poll_proceed_and_make_progress(cx)); Poll::Ready(Ok(&[])) } @@ -73,6 +75,20 @@ impl fmt::Debug for Empty { } } +cfg_coop! { + fn poll_proceed_and_make_progress(cx: &mut Context<'_>) -> Poll<()> { + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); + coop.made_progress(); + Poll::Ready(()) + } +} + +cfg_not_coop! { + fn poll_proceed_and_make_progress(_: &mut Context<'_>) -> Poll<()> { + Poll::Ready(()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/io/util/fill_buf.rs b/src/io/util/fill_buf.rs index 3655c01..bb07c76 100644 --- a/src/io/util/fill_buf.rs +++ b/src/io/util/fill_buf.rs @@ -40,6 +40,12 @@ impl<'a, R: AsyncBufRead + ?Sized + Unpin> Future for FillBuf<'a, R> { // Safety: This is necessary only due to a limitation in the // borrow checker. Once Rust starts using the polonius borrow // checker, this can be simplified. + // + // The safety of this transmute relies on the fact that the + // value of `reader` is `None` when we return in this branch. + // Otherwise the caller could poll us again after + // completion, and access the mutable reference while the + // returned immutable reference still exists. let slice = std::mem::transmute::<&[u8], &'a [u8]>(slice); Poll::Ready(Ok(slice)) }, diff --git a/src/io/util/mem.rs b/src/io/util/mem.rs index 4eefe7b..31884b3 100644 --- a/src/io/util/mem.rs +++ b/src/io/util/mem.rs @@ -177,10 +177,8 @@ impl Pipe { waker.wake(); } } -} -impl AsyncRead for Pipe { - fn poll_read( + fn poll_read_internal( mut self: Pin<&mut Self>, cx: &mut task::Context<'_>, buf: &mut ReadBuf<'_>, @@ -204,10 +202,8 @@ impl AsyncRead for Pipe { Poll::Pending } } -} -impl AsyncWrite for Pipe { - fn poll_write( + fn poll_write_internal( mut self: Pin<&mut Self>, cx: &mut task::Context<'_>, buf: &[u8], @@ -228,6 +224,62 @@ impl AsyncWrite for Pipe { } Poll::Ready(Ok(len)) } +} + +impl AsyncRead for Pipe { + cfg_coop! { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll<std::io::Result<()>> { + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); + + let ret = self.poll_read_internal(cx, buf); + if ret.is_ready() { + coop.made_progress(); + } + ret + } + } + + cfg_not_coop! { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll<std::io::Result<()>> { + self.poll_read_internal(cx, buf) + } + } +} + +impl AsyncWrite for Pipe { + cfg_coop! { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &[u8], + ) -> Poll<std::io::Result<usize>> { + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); + + let ret = self.poll_write_internal(cx, buf); + if ret.is_ready() { + coop.made_progress(); + } + ret + } + } + + cfg_not_coop! { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &[u8], + ) -> Poll<std::io::Result<usize>> { + self.poll_write_internal(cx, buf) + } + } fn poll_flush(self: Pin<&mut Self>, _: &mut task::Context<'_>) -> Poll<std::io::Result<()>> { Poll::Ready(Ok(())) diff --git a/src/io/util/read.rs b/src/io/util/read.rs index edc9d5a..a1f9c8a 100644 --- a/src/io/util/read.rs +++ b/src/io/util/read.rs @@ -48,7 +48,7 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> { let me = self.project(); - let mut buf = ReadBuf::new(*me.buf); + let mut buf = ReadBuf::new(me.buf); ready!(Pin::new(me.reader).poll_read(cx, &mut buf))?; Poll::Ready(Ok(buf.filled().len())) } diff --git a/src/io/util/read_exact.rs b/src/io/util/read_exact.rs index 1e8150e..dbdd58b 100644 --- a/src/io/util/read_exact.rs +++ b/src/io/util/read_exact.rs @@ -51,13 +51,13 @@ where type Output = io::Result<usize>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> { - let mut me = self.project(); + let me = self.project(); loop { // if our buffer is empty, then we need to read some data to continue. let rem = me.buf.remaining(); if rem != 0 { - ready!(Pin::new(&mut *me.reader).poll_read(cx, &mut me.buf))?; + ready!(Pin::new(&mut *me.reader).poll_read(cx, me.buf))?; if me.buf.remaining() == rem { return Err(eof()).into(); } diff --git a/src/io/util/take.rs b/src/io/util/take.rs index b5e90c9..df2f61b 100644 --- a/src/io/util/take.rs +++ b/src/io/util/take.rs @@ -86,7 +86,11 @@ impl<R: AsyncRead> AsyncRead for Take<R> { let me = self.project(); let mut b = buf.take(*me.limit_ as usize); + + let buf_ptr = b.filled().as_ptr(); ready!(me.inner.poll_read(cx, &mut b))?; + assert_eq!(b.filled().as_ptr(), buf_ptr); + let n = b.filled().len(); // We need to update the original ReadBuf diff --git a/src/io/util/vec_with_initialized.rs b/src/io/util/vec_with_initialized.rs index 208cc93..a9b94e3 100644 --- a/src/io/util/vec_with_initialized.rs +++ b/src/io/util/vec_with_initialized.rs @@ -1,19 +1,18 @@ use crate::io::ReadBuf; use std::mem::MaybeUninit; -mod private { - pub trait Sealed {} - - impl Sealed for Vec<u8> {} - impl Sealed for &mut Vec<u8> {} -} +/// Something that looks like a `Vec<u8>`. +/// +/// # Safety +/// +/// The implementor must guarantee that the vector returned by the +/// `as_mut` and `as_mut` methods do not change from one call to +/// another. +pub(crate) unsafe trait VecU8: AsRef<Vec<u8>> + AsMut<Vec<u8>> {} -/// A sealed trait that constrains the generic type parameter in `VecWithInitialized<V>`. That struct's safety relies -/// on certain invariants upheld by `Vec<u8>`. -pub(crate) trait VecU8: AsMut<Vec<u8>> + private::Sealed {} +unsafe impl VecU8 for Vec<u8> {} +unsafe impl VecU8 for &mut Vec<u8> {} -impl VecU8 for Vec<u8> {} -impl VecU8 for &mut Vec<u8> {} /// This struct wraps a `Vec<u8>` or `&mut Vec<u8>`, combining it with a /// `num_initialized`, which keeps track of the number of initialized bytes /// in the unused capacity. @@ -64,8 +63,8 @@ where } #[cfg(feature = "io-util")] - pub(crate) fn is_empty(&mut self) -> bool { - self.vec.as_mut().is_empty() + pub(crate) fn is_empty(&self) -> bool { + self.vec.as_ref().is_empty() } pub(crate) fn get_read_buf<'a>(&'a mut self) -> ReadBuf<'a> { diff --git a/src/io/util/write_all.rs b/src/io/util/write_all.rs index e59d41e..abd3e39 100644 --- a/src/io/util/write_all.rs +++ b/src/io/util/write_all.rs @@ -42,7 +42,7 @@ where while !me.buf.is_empty() { let n = ready!(Pin::new(&mut *me.writer).poll_write(cx, me.buf))?; { - let (_, rest) = mem::replace(&mut *me.buf, &[]).split_at(n); + let (_, rest) = mem::take(&mut *me.buf).split_at(n); *me.buf = rest; } if n == 0 { @@ -1,7 +1,9 @@ #![allow( clippy::cognitive_complexity, clippy::large_enum_variant, - clippy::needless_doctest_main + clippy::module_inception, + clippy::needless_doctest_main, + clippy::declare_interior_mutable_const )] #![warn( missing_debug_implementations, @@ -10,17 +12,13 @@ unreachable_pub )] #![deny(unused_must_use)] -#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))] #![doc(test( no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) ))] #![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(docsrs, feature(doc_cfg_hide))] -#![cfg_attr(docsrs, doc(cfg_hide(docsrs)))] -#![cfg_attr(docsrs, doc(cfg_hide(loom)))] -#![cfg_attr(docsrs, doc(cfg_hide(not(loom))))] #![cfg_attr(docsrs, allow(unused_attributes))] +#![cfg_attr(loom, allow(dead_code, unreachable_pub))] //! A runtime for writing reliable network applications without compromising speed. //! @@ -119,7 +117,7 @@ //! The [`tokio::sync`] module contains synchronization primitives to use when //! needing to communicate or share data. These include: //! -//! * channels ([`oneshot`], [`mpsc`], and [`watch`]), for sending values +//! * channels ([`oneshot`], [`mpsc`], [`watch`], and [`broadcast`]), for sending values //! between tasks, //! * a non-blocking [`Mutex`], for controlling access to a shared, mutable //! value, @@ -135,6 +133,7 @@ //! [`oneshot`]: crate::sync::oneshot //! [`mpsc`]: crate::sync::mpsc //! [`watch`]: crate::sync::watch +//! [`broadcast`]: crate::sync::broadcast //! //! The [`tokio::time`] module provides utilities for tracking time and //! scheduling work. This includes functions for setting [timeouts][timeout] for @@ -156,7 +155,7 @@ //! provide the functionality you need. //! //! Using the runtime requires the "rt" or "rt-multi-thread" feature flags, to -//! enable the basic [single-threaded scheduler][rt] and the [thread-pool +//! enable the current-thread [single-threaded scheduler][rt] and the [multi-thread //! scheduler][rt-multi-thread], respectively. See the [`runtime` module //! documentation][rt-features] for details. In addition, the "macros" feature //! flag enables the `#[tokio::main]` and `#[tokio::test]` attributes. @@ -175,12 +174,15 @@ //! swapping the currently running task on each thread. However, this kind of //! swapping can only happen at `.await` points, so code that spends a long time //! without reaching an `.await` will prevent other tasks from running. To -//! combat this, Tokio provides two kinds of threads: Core threads and blocking -//! threads. The core threads are where all asynchronous code runs, and Tokio -//! will by default spawn one for each CPU core. The blocking threads are -//! spawned on demand, can be used to run blocking code that would otherwise -//! block other tasks from running and are kept alive when not used for a certain -//! amount of time which can be configured with [`thread_keep_alive`]. +//! combat this, Tokio provides two kinds of threads: Core threads and blocking threads. +//! +//! The core threads are where all asynchronous code runs, and Tokio will by default +//! spawn one for each CPU core. You can use the environment variable `TOKIO_WORKER_THREADS` +//! to override the default value. +//! +//! The blocking threads are spawned on demand, can be used to run blocking code +//! that would otherwise block other tasks from running and are kept alive when +//! not used for a certain amount of time which can be configured with [`thread_keep_alive`]. //! Since it is not possible for Tokio to swap out blocking tasks, like it //! can do with asynchronous code, the upper limit on the number of blocking //! threads is very large. These limits can be configured on the [`Builder`]. @@ -312,8 +314,8 @@ //! Beware though that this will pull in many extra dependencies that you may not //! need. //! -//! - `full`: Enables all Tokio public API features listed below except `test-util`. -//! - `rt`: Enables `tokio::spawn`, the basic (current thread) scheduler, +//! - `full`: Enables all features listed below except `test-util` and `tracing`. +//! - `rt`: Enables `tokio::spawn`, the current-thread scheduler, //! and non-scheduler utilities. //! - `rt-multi-thread`: Enables the heavier, multi-threaded, work-stealing scheduler. //! - `io-util`: Enables the IO based `Ext` traits. @@ -329,30 +331,91 @@ //! - `signal`: Enables all `tokio::signal` types. //! - `fs`: Enables `tokio::fs` types. //! - `test-util`: Enables testing based infrastructure for the Tokio runtime. +//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's +//! synchronization primitives internally. Also, this +//! dependency is necessary to construct some of our primitives +//! in a const context. MSRV may increase according to the +//! _parking_lot_ release in use. //! //! _Note: `AsyncRead` and `AsyncWrite` traits do not require any features and are //! always available._ //! -//! ### Internal features +//! ### Unstable features //! -//! These features do not expose any new API, but influence internal -//! implementation aspects of Tokio, and can pull in additional -//! dependencies. +//! Some feature flags are only available when specifying the `tokio_unstable` flag: //! -//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's -//! synchronization primitives internally. MSRV may increase according to the -//! _parking_lot_ release in use. +//! - `tracing`: Enables tracing events. //! -//! ### Unstable features +//! Likewise, some parts of the API are only available with the same flag: //! -//! These feature flags enable **unstable** features. The public API may break in 1.x -//! releases. To enable these features, the `--cfg tokio_unstable` must be passed to -//! `rustc` when compiling. This is easiest done using the `RUSTFLAGS` env variable: -//! `RUSTFLAGS="--cfg tokio_unstable"`. +//! - [`task::Builder`] +//! - Some methods on [`task::JoinSet`] +//! - [`runtime::RuntimeMetrics`] +//! - [`runtime::Builder::unhandled_panic`] +//! - [`task::Id`] //! -//! - `tracing`: Enables tracing events. +//! This flag enables **unstable** features. The public API of these features +//! may break in 1.x releases. To enable these features, the `--cfg +//! tokio_unstable` argument must be passed to `rustc` when compiling. This +//! serves to explicitly opt-in to features which may break semver conventions, +//! since Cargo [does not yet directly support such opt-ins][unstable features]. //! +//! You can specify it in your project's `.cargo/config.toml` file: +//! +//! ```toml +//! [build] +//! rustflags = ["--cfg", "tokio_unstable"] +//! ``` +//! +//! Alternatively, you can specify it with an environment variable: +//! +//! ```sh +//! ## Many *nix shells: +//! export RUSTFLAGS="--cfg tokio_unstable" +//! cargo build +//! ``` +//! +//! ```powershell +//! ## Windows PowerShell: +//! $Env:RUSTFLAGS="--cfg tokio_unstable" +//! cargo build +//! ``` +//! +//! [unstable features]: https://internals.rust-lang.org/t/feature-request-unstable-opt-in-non-transitive-crate-features/16193#why-not-a-crate-feature-2 //! [feature flags]: https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section +//! +//! ## WASM support +//! +//! Tokio has some limited support for the WASM platform. Without the +//! `tokio_unstable` flag, the following features are supported: +//! +//! * `sync` +//! * `macros` +//! * `io-util` +//! * `rt` +//! * `time` +//! +//! Enabling any other feature (including `full`) will cause a compilation +//! failure. +//! +//! The `time` module will only work on WASM platforms that have support for +//! timers (e.g. wasm32-wasi). The timing functions will panic if used on a WASM +//! platform that does not support timers. +//! +//! Note also that if the runtime becomes indefinitely idle, it will panic +//! immediately instead of blocking forever. On platforms that don't support +//! time, this means that the runtime can never be idle in any way. +//! +//! ### Unstable WASM support +//! +//! Tokio also has unstable support for some additional WASM features. This +//! requires the use of the `tokio_unstable` flag. +//! +//! Using this flag enables the use of `tokio::net` on the wasm32-wasi target. +//! However, not all methods are available on the networking types as WASI +//! currently does not support the creation of new sockets from within WASM. +//! Because of this, sockets must currently be created via the `FromRawFd` +//! trait. // Test that pointer width is compatible. This asserts that e.g. usize is at // least 32 bits, which a lot of components in Tokio currently assumes. @@ -367,6 +430,37 @@ compile_error! { "Tokio requires the platform pointer width to be 32, 64, or 128 bits" } +// Ensure that our build script has correctly set cfg flags for wasm. +// +// Each condition is written all(a, not(b)). This should be read as +// "if a, then we must also have b". +#[cfg(any( + all(target_arch = "wasm32", not(tokio_wasm)), + all(target_arch = "wasm64", not(tokio_wasm)), + all(target_family = "wasm", not(tokio_wasm)), + all(target_os = "wasi", not(tokio_wasm)), + all(target_os = "wasi", not(tokio_wasi)), + all(target_os = "wasi", tokio_wasm_not_wasi), + all(tokio_wasm, not(any(target_arch = "wasm32", target_arch = "wasm64"))), + all(tokio_wasm_not_wasi, not(tokio_wasm)), + all(tokio_wasi, not(tokio_wasm)) +))] +compile_error!("Tokio's build script has incorrectly detected wasm."); + +#[cfg(all( + not(tokio_unstable), + tokio_wasm, + any( + feature = "fs", + feature = "io-std", + feature = "net", + feature = "process", + feature = "rt-multi-thread", + feature = "signal" + ) +))] +compile_error!("Only features sync,macros,io-util,rt,time are supported on wasm."); + // Includes re-exports used by macros. // // This module is not intended to be part of the public API. In general, any @@ -385,20 +479,25 @@ pub mod io; pub mod net; mod loom; -mod park; cfg_process! { pub mod process; } -#[cfg(any(feature = "net", feature = "fs", feature = "io-std"))] +#[cfg(any( + feature = "fs", + feature = "io-std", + feature = "net", + all(windows, feature = "process"), +))] mod blocking; cfg_rt! { pub mod runtime; } - -pub(crate) mod coop; +cfg_not_rt! { + pub(crate) mod runtime; +} cfg_signal! { pub mod signal; @@ -482,14 +581,6 @@ pub(crate) use self::doc::os; #[allow(unused)] pub(crate) use std::os; -#[cfg(docsrs)] -#[allow(unused)] -pub(crate) use self::doc::winapi; - -#[cfg(all(not(docsrs), windows, feature = "net"))] -#[allow(unused)] -pub(crate) use ::winapi; - cfg_macros! { /// Implementation detail of the `select!` macro. This macro is **not** /// intended to be used as part of the public API and is permitted to diff --git a/src/loom/mocked.rs b/src/loom/mocked.rs index 367d59b..56dc1a0 100644 --- a/src/loom/mocked.rs +++ b/src/loom/mocked.rs @@ -25,6 +25,13 @@ pub(crate) mod sync { } } pub(crate) use loom::sync::*; + + pub(crate) mod atomic { + pub(crate) use loom::sync::atomic::*; + + // TODO: implement a loom version + pub(crate) type StaticAtomicU64 = std::sync::atomic::AtomicU64; + } } pub(crate) mod rand { @@ -38,3 +45,8 @@ pub(crate) mod sys { 2 } } + +pub(crate) mod thread { + pub use loom::lazy_static::AccessError; + pub use loom::thread::*; +} diff --git a/src/loom/std/atomic_ptr.rs b/src/loom/std/atomic_ptr.rs deleted file mode 100644 index 236645f..0000000 --- a/src/loom/std/atomic_ptr.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::fmt; -use std::ops::{Deref, DerefMut}; - -/// `AtomicPtr` providing an additional `load_unsync` function. -pub(crate) struct AtomicPtr<T> { - inner: std::sync::atomic::AtomicPtr<T>, -} - -impl<T> AtomicPtr<T> { - pub(crate) fn new(ptr: *mut T) -> AtomicPtr<T> { - let inner = std::sync::atomic::AtomicPtr::new(ptr); - AtomicPtr { inner } - } -} - -impl<T> Deref for AtomicPtr<T> { - type Target = std::sync::atomic::AtomicPtr<T>; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl<T> DerefMut for AtomicPtr<T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} - -impl<T> fmt::Debug for AtomicPtr<T> { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - self.deref().fmt(fmt) - } -} diff --git a/src/loom/std/atomic_u16.rs b/src/loom/std/atomic_u16.rs index c1c5312..c9e105c 100644 --- a/src/loom/std/atomic_u16.rs +++ b/src/loom/std/atomic_u16.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt; use std::ops::Deref; -/// `AtomicU16` providing an additional `load_unsync` function. +/// `AtomicU16` providing an additional `unsync_load` function. pub(crate) struct AtomicU16 { inner: UnsafeCell<std::sync::atomic::AtomicU16>, } @@ -23,7 +23,7 @@ impl AtomicU16 { /// All mutations must have happened before the unsynchronized load. /// Additionally, there must be no concurrent mutations. pub(crate) unsafe fn unsync_load(&self) -> u16 { - *(*self.inner.get()).get_mut() + core::ptr::read(self.inner.get() as *const u16) } } diff --git a/src/loom/std/atomic_u32.rs b/src/loom/std/atomic_u32.rs index 61f95fb..ee0d2d3 100644 --- a/src/loom/std/atomic_u32.rs +++ b/src/loom/std/atomic_u32.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt; use std::ops::Deref; -/// `AtomicU32` providing an additional `load_unsync` function. +/// `AtomicU32` providing an additional `unsync_load` function. pub(crate) struct AtomicU32 { inner: UnsafeCell<std::sync::atomic::AtomicU32>, } @@ -15,6 +15,16 @@ impl AtomicU32 { let inner = UnsafeCell::new(std::sync::atomic::AtomicU32::new(val)); AtomicU32 { inner } } + + /// Performs an unsynchronized load. + /// + /// # Safety + /// + /// All mutations must have happened before the unsynchronized load. + /// Additionally, there must be no concurrent mutations. + pub(crate) unsafe fn unsync_load(&self) -> u32 { + core::ptr::read(self.inner.get() as *const u32) + } } impl Deref for AtomicU32 { diff --git a/src/loom/std/atomic_u64.rs b/src/loom/std/atomic_u64.rs index 8ea6bd4..ce391be 100644 --- a/src/loom/std/atomic_u64.rs +++ b/src/loom/std/atomic_u64.rs @@ -7,65 +7,13 @@ // `#[cfg(target_has_atomic = "64")]`. // Refs: https://github.com/rust-lang/rust/tree/master/src/librustc_target cfg_has_atomic_u64! { - pub(crate) use std::sync::atomic::AtomicU64; + #[path = "atomic_u64_native.rs"] + mod imp; } cfg_not_has_atomic_u64! { - use crate::loom::sync::Mutex; - use std::sync::atomic::Ordering; - - #[derive(Debug)] - pub(crate) struct AtomicU64 { - inner: Mutex<u64>, - } - - impl AtomicU64 { - pub(crate) fn new(val: u64) -> Self { - Self { - inner: Mutex::new(val), - } - } - - pub(crate) fn load(&self, _: Ordering) -> u64 { - *self.inner.lock() - } - - pub(crate) fn store(&self, val: u64, _: Ordering) { - *self.inner.lock() = val; - } - - pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 { - let mut lock = self.inner.lock(); - let prev = *lock; - *lock = prev | val; - prev - } - - pub(crate) fn compare_exchange( - &self, - current: u64, - new: u64, - _success: Ordering, - _failure: Ordering, - ) -> Result<u64, u64> { - let mut lock = self.inner.lock(); - - if *lock == current { - *lock = new; - Ok(current) - } else { - Err(*lock) - } - } - - pub(crate) fn compare_exchange_weak( - &self, - current: u64, - new: u64, - success: Ordering, - failure: Ordering, - ) -> Result<u64, u64> { - self.compare_exchange(current, new, success, failure) - } - } + #[path = "atomic_u64_as_mutex.rs"] + mod imp; } + +pub(crate) use imp::{AtomicU64, StaticAtomicU64}; diff --git a/src/loom/std/atomic_u64_as_mutex.rs b/src/loom/std/atomic_u64_as_mutex.rs new file mode 100644 index 0000000..9b3b6fa --- /dev/null +++ b/src/loom/std/atomic_u64_as_mutex.rs @@ -0,0 +1,76 @@ +use crate::loom::sync::Mutex; +use std::sync::atomic::Ordering; + +cfg_has_const_mutex_new! { + #[path = "atomic_u64_static_const_new.rs"] + mod static_macro; +} + +cfg_not_has_const_mutex_new! { + #[path = "atomic_u64_static_once_cell.rs"] + mod static_macro; +} + +pub(crate) use static_macro::StaticAtomicU64; + +#[derive(Debug)] +pub(crate) struct AtomicU64 { + inner: Mutex<u64>, +} + +impl AtomicU64 { + pub(crate) fn load(&self, _: Ordering) -> u64 { + *self.inner.lock() + } + + pub(crate) fn store(&self, val: u64, _: Ordering) { + *self.inner.lock() = val; + } + + pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 { + let mut lock = self.inner.lock(); + let prev = *lock; + *lock = prev + val; + prev + } + + pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 { + let mut lock = self.inner.lock(); + let prev = *lock; + *lock = prev | val; + prev + } + + pub(crate) fn compare_exchange( + &self, + current: u64, + new: u64, + _success: Ordering, + _failure: Ordering, + ) -> Result<u64, u64> { + let mut lock = self.inner.lock(); + + if *lock == current { + *lock = new; + Ok(current) + } else { + Err(*lock) + } + } + + pub(crate) fn compare_exchange_weak( + &self, + current: u64, + new: u64, + success: Ordering, + failure: Ordering, + ) -> Result<u64, u64> { + self.compare_exchange(current, new, success, failure) + } +} + +impl Default for AtomicU64 { + fn default() -> AtomicU64 { + AtomicU64::new(u64::default()) + } +} diff --git a/src/loom/std/atomic_u64_native.rs b/src/loom/std/atomic_u64_native.rs new file mode 100644 index 0000000..08adb28 --- /dev/null +++ b/src/loom/std/atomic_u64_native.rs @@ -0,0 +1,4 @@ +pub(crate) use std::sync::atomic::{AtomicU64, Ordering}; + +/// Alias `AtomicU64` to `StaticAtomicU64` +pub(crate) type StaticAtomicU64 = AtomicU64; diff --git a/src/loom/std/atomic_u64_static_const_new.rs b/src/loom/std/atomic_u64_static_const_new.rs new file mode 100644 index 0000000..a421534 --- /dev/null +++ b/src/loom/std/atomic_u64_static_const_new.rs @@ -0,0 +1,12 @@ +use super::AtomicU64; +use crate::loom::sync::Mutex; + +pub(crate) type StaticAtomicU64 = AtomicU64; + +impl AtomicU64 { + pub(crate) const fn new(val: u64) -> Self { + Self { + inner: Mutex::const_new(val), + } + } +} diff --git a/src/loom/std/atomic_u64_static_once_cell.rs b/src/loom/std/atomic_u64_static_once_cell.rs new file mode 100644 index 0000000..40c6172 --- /dev/null +++ b/src/loom/std/atomic_u64_static_once_cell.rs @@ -0,0 +1,57 @@ +use super::AtomicU64; +use crate::loom::sync::{atomic::Ordering, Mutex}; +use crate::util::once_cell::OnceCell; + +pub(crate) struct StaticAtomicU64 { + init: u64, + cell: OnceCell<Mutex<u64>>, +} + +impl AtomicU64 { + pub(crate) fn new(val: u64) -> Self { + Self { + inner: Mutex::new(val), + } + } +} + +impl StaticAtomicU64 { + pub(crate) const fn new(val: u64) -> StaticAtomicU64 { + StaticAtomicU64 { + init: val, + cell: OnceCell::new(), + } + } + + pub(crate) fn load(&self, order: Ordering) -> u64 { + *self.inner().lock() + } + + pub(crate) fn fetch_add(&self, val: u64, order: Ordering) -> u64 { + let mut lock = self.inner().lock(); + let prev = *lock; + *lock = prev + val; + prev + } + + pub(crate) fn compare_exchange_weak( + &self, + current: u64, + new: u64, + _success: Ordering, + _failure: Ordering, + ) -> Result<u64, u64> { + let mut lock = self.inner().lock(); + + if *lock == current { + *lock = new; + Ok(current) + } else { + Err(*lock) + } + } + + fn inner(&self) -> &Mutex<u64> { + self.cell.get(|| Mutex::new(self.init)) + } +} diff --git a/src/loom/std/atomic_u8.rs b/src/loom/std/atomic_u8.rs deleted file mode 100644 index 408aea3..0000000 --- a/src/loom/std/atomic_u8.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::cell::UnsafeCell; -use std::fmt; -use std::ops::Deref; - -/// `AtomicU8` providing an additional `load_unsync` function. -pub(crate) struct AtomicU8 { - inner: UnsafeCell<std::sync::atomic::AtomicU8>, -} - -unsafe impl Send for AtomicU8 {} -unsafe impl Sync for AtomicU8 {} - -impl AtomicU8 { - pub(crate) const fn new(val: u8) -> AtomicU8 { - let inner = UnsafeCell::new(std::sync::atomic::AtomicU8::new(val)); - AtomicU8 { inner } - } -} - -impl Deref for AtomicU8 { - type Target = std::sync::atomic::AtomicU8; - - fn deref(&self) -> &Self::Target { - // safety: it is always safe to access `&self` fns on the inner value as - // we never perform unsafe mutations. - unsafe { &*self.inner.get() } - } -} - -impl fmt::Debug for AtomicU8 { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - self.deref().fmt(fmt) - } -} diff --git a/src/loom/std/atomic_usize.rs b/src/loom/std/atomic_usize.rs index 0d5f36e..c5503a2 100644 --- a/src/loom/std/atomic_usize.rs +++ b/src/loom/std/atomic_usize.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt; use std::ops; -/// `AtomicUsize` providing an additional `load_unsync` function. +/// `AtomicUsize` providing an additional `unsync_load` function. pub(crate) struct AtomicUsize { inner: UnsafeCell<std::sync::atomic::AtomicUsize>, } @@ -23,7 +23,7 @@ impl AtomicUsize { /// All mutations must have happened before the unsynchronized load. /// Additionally, there must be no concurrent mutations. pub(crate) unsafe fn unsync_load(&self) -> usize { - *(*self.inner.get()).get_mut() + core::ptr::read(self.inner.get() as *const usize) } pub(crate) fn with_mut<R>(&mut self, f: impl FnOnce(&mut usize) -> R) -> R { diff --git a/src/loom/std/mod.rs b/src/loom/std/mod.rs index 8b6e8bc..6bd1ad9 100644 --- a/src/loom/std/mod.rs +++ b/src/loom/std/mod.rs @@ -1,10 +1,8 @@ #![cfg_attr(any(not(feature = "full"), loom), allow(unused_imports, dead_code))] -mod atomic_ptr; mod atomic_u16; mod atomic_u32; mod atomic_u64; -mod atomic_u8; mod atomic_usize; mod mutex; #[cfg(feature = "parking_lot")] @@ -25,6 +23,10 @@ pub(crate) mod future { pub(crate) use crate::sync::AtomicWaker; } +pub(crate) mod hint { + pub(crate) use std::hint::spin_loop; +} + pub(crate) mod rand { use std::collections::hash_map::RandomState; use std::hash::{BuildHasher, Hash, Hasher}; @@ -67,24 +69,39 @@ pub(crate) mod sync { pub(crate) use crate::loom::std::mutex::Mutex; pub(crate) mod atomic { - pub(crate) use crate::loom::std::atomic_ptr::AtomicPtr; pub(crate) use crate::loom::std::atomic_u16::AtomicU16; pub(crate) use crate::loom::std::atomic_u32::AtomicU32; - pub(crate) use crate::loom::std::atomic_u64::AtomicU64; - pub(crate) use crate::loom::std::atomic_u8::AtomicU8; + pub(crate) use crate::loom::std::atomic_u64::{AtomicU64, StaticAtomicU64}; pub(crate) use crate::loom::std::atomic_usize::AtomicUsize; - pub(crate) use std::sync::atomic::{fence, AtomicBool, Ordering}; - // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead. - #[allow(deprecated)] - pub(crate) use std::sync::atomic::spin_loop_hint; + pub(crate) use std::sync::atomic::{fence, AtomicBool, AtomicPtr, AtomicU8, Ordering}; } } pub(crate) mod sys { #[cfg(feature = "rt-multi-thread")] pub(crate) fn num_cpus() -> usize { - usize::max(1, num_cpus::get()) + const ENV_WORKER_THREADS: &str = "TOKIO_WORKER_THREADS"; + + match std::env::var(ENV_WORKER_THREADS) { + Ok(s) => { + let n = s.parse().unwrap_or_else(|e| { + panic!( + "\"{}\" must be usize, error: {}, value: {}", + ENV_WORKER_THREADS, e, s + ) + }); + assert!(n > 0, "\"{}\" cannot be set to 0", ENV_WORKER_THREADS); + n + } + Err(std::env::VarError::NotPresent) => usize::max(1, num_cpus::get()), + Err(std::env::VarError::NotUnicode(e)) => { + panic!( + "\"{}\" must be valid unicode, error: {:?}", + ENV_WORKER_THREADS, e + ) + } + } } #[cfg(not(feature = "rt-multi-thread"))] @@ -96,14 +113,12 @@ pub(crate) mod sys { pub(crate) mod thread { #[inline] pub(crate) fn yield_now() { - // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead. - #[allow(deprecated)] - std::sync::atomic::spin_loop_hint(); + std::hint::spin_loop(); } #[allow(unused_imports)] pub(crate) use std::thread::{ - current, panicking, park, park_timeout, sleep, spawn, Builder, JoinHandle, LocalKey, - Result, Thread, ThreadId, + current, panicking, park, park_timeout, sleep, spawn, AccessError, Builder, JoinHandle, + LocalKey, Result, Thread, ThreadId, }; } diff --git a/src/loom/std/mutex.rs b/src/loom/std/mutex.rs index 3f686e0..076f786 100644 --- a/src/loom/std/mutex.rs +++ b/src/loom/std/mutex.rs @@ -13,6 +13,12 @@ impl<T> Mutex<T> { } #[inline] + #[cfg(not(tokio_no_const_mutex_new))] + pub(crate) const fn const_new(t: T) -> Mutex<T> { + Mutex(sync::Mutex::new(t)) + } + + #[inline] pub(crate) fn lock(&self) -> MutexGuard<'_, T> { match self.0.lock() { Ok(guard) => guard, diff --git a/src/loom/std/parking_lot.rs b/src/loom/std/parking_lot.rs index 8448bed..e3af258 100644 --- a/src/loom/std/parking_lot.rs +++ b/src/loom/std/parking_lot.rs @@ -3,83 +3,143 @@ //! //! This can be extended to additional types/methods as required. +use std::fmt; +use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; use std::sync::LockResult; use std::time::Duration; +// All types in this file are marked with PhantomData to ensure that +// parking_lot's send_guard feature does not leak through and affect when Tokio +// types are Send. +// +// See <https://github.com/tokio-rs/tokio/pull/4359> for more info. + // Types that do not need wrapping -pub(crate) use parking_lot::{MutexGuard, RwLockReadGuard, RwLockWriteGuard, WaitTimeoutResult}; +pub(crate) use parking_lot::WaitTimeoutResult; + +#[derive(Debug)] +pub(crate) struct Mutex<T: ?Sized>(PhantomData<std::sync::Mutex<T>>, parking_lot::Mutex<T>); + +#[derive(Debug)] +pub(crate) struct RwLock<T>(PhantomData<std::sync::RwLock<T>>, parking_lot::RwLock<T>); + +#[derive(Debug)] +pub(crate) struct Condvar(PhantomData<std::sync::Condvar>, parking_lot::Condvar); -/// Adapter for `parking_lot::Mutex` to the `std::sync::Mutex` interface. #[derive(Debug)] -pub(crate) struct Mutex<T: ?Sized>(parking_lot::Mutex<T>); +pub(crate) struct MutexGuard<'a, T: ?Sized>( + PhantomData<std::sync::MutexGuard<'a, T>>, + parking_lot::MutexGuard<'a, T>, +); #[derive(Debug)] -pub(crate) struct RwLock<T>(parking_lot::RwLock<T>); +pub(crate) struct RwLockReadGuard<'a, T: ?Sized>( + PhantomData<std::sync::RwLockReadGuard<'a, T>>, + parking_lot::RwLockReadGuard<'a, T>, +); -/// Adapter for `parking_lot::Condvar` to the `std::sync::Condvar` interface. #[derive(Debug)] -pub(crate) struct Condvar(parking_lot::Condvar); +pub(crate) struct RwLockWriteGuard<'a, T: ?Sized>( + PhantomData<std::sync::RwLockWriteGuard<'a, T>>, + parking_lot::RwLockWriteGuard<'a, T>, +); impl<T> Mutex<T> { #[inline] pub(crate) fn new(t: T) -> Mutex<T> { - Mutex(parking_lot::Mutex::new(t)) + Mutex(PhantomData, parking_lot::Mutex::new(t)) } #[inline] - #[cfg(all(feature = "parking_lot", not(all(loom, test)),))] + #[cfg(all(feature = "parking_lot", not(all(loom, test))))] #[cfg_attr(docsrs, doc(cfg(all(feature = "parking_lot",))))] pub(crate) const fn const_new(t: T) -> Mutex<T> { - Mutex(parking_lot::const_mutex(t)) + Mutex(PhantomData, parking_lot::const_mutex(t)) } #[inline] pub(crate) fn lock(&self) -> MutexGuard<'_, T> { - self.0.lock() + MutexGuard(PhantomData, self.1.lock()) } #[inline] pub(crate) fn try_lock(&self) -> Option<MutexGuard<'_, T>> { - self.0.try_lock() + self.1 + .try_lock() + .map(|guard| MutexGuard(PhantomData, guard)) } #[inline] pub(crate) fn get_mut(&mut self) -> &mut T { - self.0.get_mut() + self.1.get_mut() } // Note: Additional methods `is_poisoned` and `into_inner`, can be // provided here as needed. } +impl<'a, T: ?Sized> Deref for MutexGuard<'a, T> { + type Target = T; + fn deref(&self) -> &T { + self.1.deref() + } +} + +impl<'a, T: ?Sized> DerefMut for MutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut T { + self.1.deref_mut() + } +} + impl<T> RwLock<T> { pub(crate) fn new(t: T) -> RwLock<T> { - RwLock(parking_lot::RwLock::new(t)) + RwLock(PhantomData, parking_lot::RwLock::new(t)) } pub(crate) fn read(&self) -> LockResult<RwLockReadGuard<'_, T>> { - Ok(self.0.read()) + Ok(RwLockReadGuard(PhantomData, self.1.read())) } pub(crate) fn write(&self) -> LockResult<RwLockWriteGuard<'_, T>> { - Ok(self.0.write()) + Ok(RwLockWriteGuard(PhantomData, self.1.write())) + } +} + +impl<'a, T: ?Sized> Deref for RwLockReadGuard<'a, T> { + type Target = T; + fn deref(&self) -> &T { + self.1.deref() + } +} + +impl<'a, T: ?Sized> Deref for RwLockWriteGuard<'a, T> { + type Target = T; + fn deref(&self) -> &T { + self.1.deref() + } +} + +impl<'a, T: ?Sized> DerefMut for RwLockWriteGuard<'a, T> { + fn deref_mut(&mut self) -> &mut T { + self.1.deref_mut() } } impl Condvar { #[inline] pub(crate) fn new() -> Condvar { - Condvar(parking_lot::Condvar::new()) + Condvar(PhantomData, parking_lot::Condvar::new()) } #[inline] pub(crate) fn notify_one(&self) { - self.0.notify_one(); + self.1.notify_one(); } #[inline] pub(crate) fn notify_all(&self) { - self.0.notify_all(); + self.1.notify_all(); } #[inline] @@ -87,7 +147,7 @@ impl Condvar { &self, mut guard: MutexGuard<'a, T>, ) -> LockResult<MutexGuard<'a, T>> { - self.0.wait(&mut guard); + self.1.wait(&mut guard.1); Ok(guard) } @@ -97,10 +157,28 @@ impl Condvar { mut guard: MutexGuard<'a, T>, timeout: Duration, ) -> LockResult<(MutexGuard<'a, T>, WaitTimeoutResult)> { - let wtr = self.0.wait_for(&mut guard, timeout); + let wtr = self.1.wait_for(&mut guard.1, timeout); Ok((guard, wtr)) } // Note: Additional methods `wait_timeout_ms`, `wait_timeout_until`, // `wait_until` can be provided here as needed. } + +impl<'a, T: ?Sized + fmt::Display> fmt::Display for MutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.1, f) + } +} + +impl<'a, T: ?Sized + fmt::Display> fmt::Display for RwLockReadGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.1, f) + } +} + +impl<'a, T: ?Sized + fmt::Display> fmt::Display for RwLockWriteGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.1, f) + } +} diff --git a/src/macros/addr_of.rs b/src/macros/addr_of.rs new file mode 100644 index 0000000..51d4889 --- /dev/null +++ b/src/macros/addr_of.rs @@ -0,0 +1,53 @@ +//! This module defines a macro that lets you go from a raw pointer to a struct +//! to a raw pointer to a field of the struct. + +#[cfg(not(tokio_no_addr_of))] +macro_rules! generate_addr_of_methods { + ( + impl<$($gen:ident)*> $struct_name:ty {$( + $(#[$attrs:meta])* + $vis:vis unsafe fn $fn_name:ident(self: NonNull<Self>) -> NonNull<$field_type:ty> { + &self$(.$field_name:tt)+ + } + )*} + ) => { + impl<$($gen)*> $struct_name {$( + $(#[$attrs])* + $vis unsafe fn $fn_name(me: ::core::ptr::NonNull<Self>) -> ::core::ptr::NonNull<$field_type> { + let me = me.as_ptr(); + let field = ::std::ptr::addr_of_mut!((*me) $(.$field_name)+ ); + ::core::ptr::NonNull::new_unchecked(field) + } + )*} + }; +} + +// The `addr_of_mut!` macro is only available for MSRV at least 1.51.0. This +// version of the macro uses a workaround for older versions of rustc. +#[cfg(tokio_no_addr_of)] +macro_rules! generate_addr_of_methods { + ( + impl<$($gen:ident)*> $struct_name:ty {$( + $(#[$attrs:meta])* + $vis:vis unsafe fn $fn_name:ident(self: NonNull<Self>) -> NonNull<$field_type:ty> { + &self$(.$field_name:tt)+ + } + )*} + ) => { + impl<$($gen)*> $struct_name {$( + $(#[$attrs])* + $vis unsafe fn $fn_name(me: ::core::ptr::NonNull<Self>) -> ::core::ptr::NonNull<$field_type> { + let me = me.as_ptr(); + let me_u8 = me as *mut u8; + + let field_offset = { + let me_ref = &*me; + let field_ref_u8 = (&me_ref $(.$field_name)+ ) as *const $field_type as *const u8; + field_ref_u8.offset_from(me_u8) + }; + + ::core::ptr::NonNull::new_unchecked(me_u8.offset(field_offset).cast()) + } + )*} + }; +} diff --git a/src/macros/cfg.rs b/src/macros/cfg.rs index 606bce7..1c66d24 100644 --- a/src/macros/cfg.rs +++ b/src/macros/cfg.rs @@ -61,6 +61,7 @@ macro_rules! cfg_fs { ($($item:item)*) => { $( #[cfg(feature = "fs")] + #[cfg(not(tokio_wasi))] #[cfg_attr(docsrs, doc(cfg(feature = "fs")))] $item )* @@ -69,7 +70,11 @@ macro_rules! cfg_fs { macro_rules! cfg_io_blocking { ($($item:item)*) => { - $( #[cfg(any(feature = "io-std", feature = "fs"))] $item )* + $( #[cfg(any( + feature = "io-std", + feature = "fs", + all(windows, feature = "process"), + ))] $item )* } } @@ -78,12 +83,12 @@ macro_rules! cfg_io_driver { $( #[cfg(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), ))] #[cfg_attr(docsrs, doc(cfg(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), ))))] $item @@ -96,10 +101,9 @@ macro_rules! cfg_io_driver_impl { $( #[cfg(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), ))] - #[cfg_attr(docsrs, doc(cfg(all())))] $item )* } @@ -110,7 +114,7 @@ macro_rules! cfg_not_io_driver { $( #[cfg(not(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), )))] $item @@ -175,20 +179,38 @@ macro_rules! cfg_macros { } } -macro_rules! cfg_stats { +macro_rules! cfg_metrics { ($($item:item)*) => { $( - #[cfg(all(tokio_unstable, feature = "stats"))] - #[cfg_attr(docsrs, doc(cfg(feature = "stats")))] + // For now, metrics is only disabled in loom tests. + // When stabilized, it might have a dedicated feature flag. + #[cfg(all(tokio_unstable, not(loom)))] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] $item )* } } -macro_rules! cfg_not_stats { +macro_rules! cfg_not_metrics { ($($item:item)*) => { $( - #[cfg(not(all(tokio_unstable, feature = "stats")))] + #[cfg(not(all(tokio_unstable, not(loom))))] + $item + )* + } +} + +macro_rules! cfg_not_rt_and_metrics_and_net { + ($($item:item)*) => { + $( #[cfg(not(all(feature = "net", feature = "rt", all(tokio_unstable, not(loom)))))]$item )* + } +} + +macro_rules! cfg_net_or_process { + ($($item:item)*) => { + $( + #[cfg(any(feature = "net", feature = "process"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "net", feature = "process"))))] $item )* } @@ -230,6 +252,7 @@ macro_rules! cfg_process { #[cfg(feature = "process")] #[cfg_attr(docsrs, doc(cfg(feature = "process")))] #[cfg(not(loom))] + #[cfg(not(tokio_wasi))] $item )* } @@ -258,6 +281,7 @@ macro_rules! cfg_signal { #[cfg(feature = "signal")] #[cfg_attr(docsrs, doc(cfg(feature = "signal")))] #[cfg(not(loom))] + #[cfg(not(tokio_wasi))] $item )* } @@ -273,6 +297,13 @@ macro_rules! cfg_signal_internal { } } +macro_rules! cfg_signal_internal_and_unix { + ($($item:item)*) => { + #[cfg(unix)] + cfg_signal_internal! { $($item)* } + } +} + macro_rules! cfg_not_signal_internal { ($($item:item)*) => { $( @@ -317,7 +348,7 @@ macro_rules! cfg_not_rt { macro_rules! cfg_rt_multi_thread { ($($item:item)*) => { $( - #[cfg(feature = "rt-multi-thread")] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] $item )* @@ -366,10 +397,20 @@ macro_rules! cfg_trace { ($($item:item)*) => { $( #[cfg(all(tokio_unstable, feature = "tracing"))] - #[cfg_attr(docsrs, doc(cfg(feature = "tracing")))] + #[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] $item )* - } + }; +} + +macro_rules! cfg_unstable { + ($($item:item)*) => { + $( + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + $item + )* + }; } macro_rules! cfg_not_trace { @@ -420,12 +461,14 @@ macro_rules! cfg_not_coop { macro_rules! cfg_has_atomic_u64 { ($($item:item)*) => { $( - #[cfg(not(any( - target_arch = "arm", - target_arch = "mips", - target_arch = "powerpc", - target_arch = "riscv32" - )))] + #[cfg_attr( + not(tokio_no_target_has_atomic), + cfg(all(target_has_atomic = "64", not(tokio_no_atomic_u64)) + ))] + #[cfg_attr( + tokio_no_target_has_atomic, + cfg(not(tokio_no_atomic_u64)) + )] $item )* } @@ -434,12 +477,62 @@ macro_rules! cfg_has_atomic_u64 { macro_rules! cfg_not_has_atomic_u64 { ($($item:item)*) => { $( - #[cfg(any( - target_arch = "arm", - target_arch = "mips", - target_arch = "powerpc", - target_arch = "riscv32" - ))] + #[cfg_attr( + not(tokio_no_target_has_atomic), + cfg(any(not(target_has_atomic = "64"), tokio_no_atomic_u64) + ))] + #[cfg_attr( + tokio_no_target_has_atomic, + cfg(tokio_no_atomic_u64) + )] + $item + )* + } +} + +macro_rules! cfg_has_const_mutex_new { + ($($item:item)*) => { + $( + #[cfg(all( + not(all(loom, test)), + any( + feature = "parking_lot", + not(tokio_no_const_mutex_new) + ) + ))] + $item + )* + } +} + +macro_rules! cfg_not_has_const_mutex_new { + ($($item:item)*) => { + $( + #[cfg(not(all( + not(all(loom, test)), + any( + feature = "parking_lot", + not(tokio_no_const_mutex_new) + ) + )))] + $item + )* + } +} + +macro_rules! cfg_not_wasi { + ($($item:item)*) => { + $( + #[cfg(not(tokio_wasi))] + $item + )* + } +} + +macro_rules! cfg_is_wasm_not_wasi { + ($($item:item)*) => { + $( + #[cfg(tokio_wasm_not_wasi)] $item )* } diff --git a/src/macros/join.rs b/src/macros/join.rs index f91b5f1..7e85203 100644 --- a/src/macros/join.rs +++ b/src/macros/join.rs @@ -12,7 +12,7 @@ /// for **all** branches complete regardless if any complete with `Err`. Use /// [`try_join!`] to return early when `Err` is encountered. /// -/// [`try_join!`]: macro@try_join +/// [`try_join!`]: crate::try_join /// /// # Notes /// @@ -60,6 +60,9 @@ macro_rules! join { // normalization is complete. ( $($count:tt)* ) + // The expression `0+1+1+ ... +1` equal to the number of branches. + ( $($total:tt)* ) + // Normalized join! branches $( ( $($skip:tt)* ) $e:expr, )* @@ -69,24 +72,66 @@ macro_rules! join { // Safety: nothing must be moved out of `futures`. This is to satisfy // the requirement of `Pin::new_unchecked` called below. + // + // We can't use the `pin!` macro for this because `futures` is a tuple + // and the standard library provides no way to pin-project to the fields + // of a tuple. let mut futures = ( $( maybe_done($e), )* ); + // This assignment makes sure that the `poll_fn` closure only has a + // reference to the futures, instead of taking ownership of them. This + // mitigates the issue described in + // <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484> + let mut futures = &mut futures; + + // Each time the future created by poll_fn is polled, a different future will be polled first + // to ensure every future passed to join! gets a chance to make progress even if + // one of the futures consumes the whole budget. + // + // This is number of futures that will be skipped in the first loop + // iteration the next time. + let mut skip_next_time: u32 = 0; + poll_fn(move |cx| { + const COUNT: u32 = $($total)*; + let mut is_pending = false; + let mut to_run = COUNT; + + // The number of futures that will be skipped in the first loop iteration. + let mut skip = skip_next_time; + + skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 }; + + // This loop runs twice and the first `skip` futures + // are not polled in the first iteration. + loop { $( - // Extract the future for this branch from the tuple. - let ( $($skip,)* fut, .. ) = &mut futures; + if skip == 0 { + if to_run == 0 { + // Every future has been polled + break; + } + to_run -= 1; - // Safety: future is stored on the stack above - // and never moved. - let mut fut = unsafe { Pin::new_unchecked(fut) }; + // Extract the future for this branch from the tuple. + let ( $($skip,)* fut, .. ) = &mut *futures; + + // Safety: future is stored on the stack above + // and never moved. + let mut fut = unsafe { Pin::new_unchecked(fut) }; - // Try polling - if fut.poll(cx).is_pending() { - is_pending = true; + // Try polling + if fut.poll(cx).is_pending() { + is_pending = true; + } + } else { + // Future skipped, one less future to skip in the next iteration + skip -= 1; } )* + } if is_pending { Pending @@ -107,13 +152,13 @@ macro_rules! join { // ===== Normalize ===== - (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { - $crate::join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*) + (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { + $crate::join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*) }; // ===== Entry point ===== ( $($e:expr),* $(,)?) => { - $crate::join!(@{ () } $($e,)*) + $crate::join!(@{ () (0) } $($e,)*) }; } diff --git a/src/macros/mod.rs b/src/macros/mod.rs index a1839c8..57678c6 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -15,6 +15,9 @@ mod ready; #[macro_use] mod thread_local; +#[macro_use] +mod addr_of; + cfg_trace! { #[macro_use] mod trace; diff --git a/src/macros/scoped_tls.rs b/src/macros/scoped_tls.rs index f2504cb..ed74b32 100644 --- a/src/macros/scoped_tls.rs +++ b/src/macros/scoped_tls.rs @@ -10,7 +10,7 @@ macro_rules! scoped_thread_local { $vis static $name: $crate::macros::scoped_tls::ScopedKey<$ty> = $crate::macros::scoped_tls::ScopedKey { inner: { - thread_local!(static FOO: ::std::cell::Cell<*const ()> = { + tokio_thread_local!(static FOO: ::std::cell::Cell<*const ()> = const { std::cell::Cell::new(::std::ptr::null()) }); &FOO diff --git a/src/macros/select.rs b/src/macros/select.rs index 051f8cb..7ba04a0 100644 --- a/src/macros/select.rs +++ b/src/macros/select.rs @@ -101,6 +101,7 @@ /// * [`tokio::sync::watch::Receiver::changed`](crate::sync::watch::Receiver::changed) /// * [`tokio::net::TcpListener::accept`](crate::net::TcpListener::accept) /// * [`tokio::net::UnixListener::accept`](crate::net::UnixListener::accept) +/// * [`tokio::signal::unix::Signal::recv`](crate::signal::unix::Signal::recv) /// * [`tokio::io::AsyncReadExt::read`](crate::io::AsyncReadExt::read) on any `AsyncRead` /// * [`tokio::io::AsyncReadExt::read_buf`](crate::io::AsyncReadExt::read_buf) on any `AsyncRead` /// * [`tokio::io::AsyncWriteExt::write`](crate::io::AsyncWriteExt::write) on any `AsyncWrite` @@ -429,7 +430,8 @@ macro_rules! select { // // This module is defined within a scope and should not leak out of this // macro. - mod util { + #[doc(hidden)] + mod __tokio_select_util { // Generate an enum with one variant per select branch $crate::select_priv_declare_output_enum!( ( $($count)* ) ); } @@ -442,13 +444,13 @@ macro_rules! select { const BRANCHES: u32 = $crate::count!( $($count)* ); - let mut disabled: util::Mask = Default::default(); + let mut disabled: __tokio_select_util::Mask = Default::default(); // First, invoke all the pre-conditions. For any that return true, // set the appropriate bit in `disabled`. $( if !$c { - let mask: util::Mask = 1 << $crate::count!( $($skip)* ); + let mask: __tokio_select_util::Mask = 1 << $crate::count!( $($skip)* ); disabled |= mask; } )* @@ -458,8 +460,18 @@ macro_rules! select { let mut output = { // Safety: Nothing must be moved out of `futures`. This is to // satisfy the requirement of `Pin::new_unchecked` called below. + // + // We can't use the `pin!` macro for this because `futures` is a + // tuple and the standard library provides no way to pin-project to + // the fields of a tuple. let mut futures = ( $( $fut , )+ ); + // This assignment makes sure that the `poll_fn` closure only has a + // reference to the futures, instead of taking ownership of them. + // This mitigates the issue described in + // <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484> + let mut futures = &mut futures; + $crate::macros::support::poll_fn(|cx| { // Track if any branch returns pending. If no branch completes // **or** returns pending, this implies that all branches are @@ -495,7 +507,7 @@ macro_rules! select { // Extract the future for this branch from the // tuple - let ( $($skip,)* fut, .. ) = &mut futures; + let ( $($skip,)* fut, .. ) = &mut *futures; // Safety: future is stored on the stack above // and never moved. @@ -525,7 +537,7 @@ macro_rules! select { } // The select is complete, return the value - return Ready($crate::select_variant!(util::Out, ($($skip)*))(out)); + return Ready($crate::select_variant!(__tokio_select_util::Out, ($($skip)*))(out)); } )* _ => unreachable!("reaching this means there probably is an off by one bug"), @@ -536,16 +548,16 @@ macro_rules! select { Pending } else { // All branches have been disabled. - Ready(util::Out::Disabled) + Ready(__tokio_select_util::Out::Disabled) } }).await }; match output { $( - $crate::select_variant!(util::Out, ($($skip)*) ($bind)) => $handle, + $crate::select_variant!(__tokio_select_util::Out, ($($skip)*) ($bind)) => $handle, )* - util::Out::Disabled => $else, + __tokio_select_util::Out::Disabled => $else, _ => unreachable!("failed to match bind"), } }}; @@ -801,6 +813,9 @@ macro_rules! count { (_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _) => { 63 }; + (_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _) => { + 64 + }; } #[macro_export] diff --git a/src/macros/support.rs b/src/macros/support.rs index 7f11bc6..10526bc 100644 --- a/src/macros/support.rs +++ b/src/macros/support.rs @@ -1,7 +1,11 @@ cfg_macros! { pub use crate::future::poll_fn; pub use crate::future::maybe_done::maybe_done; - pub use crate::util::thread_rng_n; + + #[doc(hidden)] + pub fn thread_rng_n(n: u32) -> u32 { + crate::runtime::context::thread_rng_n(n) + } } pub use std::future::Future; diff --git a/src/macros/thread_local.rs b/src/macros/thread_local.rs index d848947..74be99a 100644 --- a/src/macros/thread_local.rs +++ b/src/macros/thread_local.rs @@ -1,4 +1,32 @@ #[cfg(all(loom, test))] -macro_rules! thread_local { +macro_rules! tokio_thread_local { + ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => { + loom::thread_local! { + $(#[$attrs])* + $vis static $name: $ty = $expr; + } + }; + ($($tts:tt)+) => { loom::thread_local!{ $($tts)+ } } } + +#[cfg(not(tokio_no_const_thread_local))] +#[cfg(not(all(loom, test)))] +macro_rules! tokio_thread_local { + ($($tts:tt)+) => { + ::std::thread_local!{ $($tts)+ } + } +} + +#[cfg(tokio_no_const_thread_local)] +#[cfg(not(all(loom, test)))] +macro_rules! tokio_thread_local { + ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => { + ::std::thread_local! { + $(#[$attrs])* + $vis static $name: $ty = $expr; + } + }; + + ($($tts:tt)+) => { ::std::thread_local!{ $($tts)+ } } +} diff --git a/src/macros/trace.rs b/src/macros/trace.rs index 31dde2f..80a257e 100644 --- a/src/macros/trace.rs +++ b/src/macros/trace.rs @@ -1,9 +1,8 @@ cfg_trace! { macro_rules! trace_op { - ($name:literal, $readiness:literal, $parent:expr) => { + ($name:expr, $readiness:literal) => { tracing::trace!( target: "runtime::resource::poll_op", - parent: $parent, op_name = $name, is_ready = $readiness ); @@ -11,14 +10,14 @@ cfg_trace! { } macro_rules! trace_poll_op { - ($name:literal, $poll:expr, $parent:expr $(,)*) => { + ($name:expr, $poll:expr $(,)*) => { match $poll { std::task::Poll::Ready(t) => { - trace_op!($name, true, $parent); + trace_op!($name, true); std::task::Poll::Ready(t) } std::task::Poll::Pending => { - trace_op!($name, false, $parent); + trace_op!($name, false); return std::task::Poll::Pending; } } diff --git a/src/macros/try_join.rs b/src/macros/try_join.rs index 6d3a893..597cd5d 100644 --- a/src/macros/try_join.rs +++ b/src/macros/try_join.rs @@ -106,6 +106,9 @@ macro_rules! try_join { // normalization is complete. ( $($count:tt)* ) + // The expression `0+1+1+ ... +1` equal to the number of branches. + ( $($total:tt)* ) + // Normalized try_join! branches $( ( $($skip:tt)* ) $e:expr, )* @@ -115,26 +118,68 @@ macro_rules! try_join { // Safety: nothing must be moved out of `futures`. This is to satisfy // the requirement of `Pin::new_unchecked` called below. + // + // We can't use the `pin!` macro for this because `futures` is a tuple + // and the standard library provides no way to pin-project to the fields + // of a tuple. let mut futures = ( $( maybe_done($e), )* ); + // This assignment makes sure that the `poll_fn` closure only has a + // reference to the futures, instead of taking ownership of them. This + // mitigates the issue described in + // <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484> + let mut futures = &mut futures; + + // Each time the future created by poll_fn is polled, a different future will be polled first + // to ensure every future passed to join! gets a chance to make progress even if + // one of the futures consumes the whole budget. + // + // This is number of futures that will be skipped in the first loop + // iteration the next time. + let mut skip_next_time: u32 = 0; + poll_fn(move |cx| { + const COUNT: u32 = $($total)*; + let mut is_pending = false; + let mut to_run = COUNT; + + // The number of futures that will be skipped in the first loop iteration + let mut skip = skip_next_time; + + skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 }; + + // This loop runs twice and the first `skip` futures + // are not polled in the first iteration. + loop { $( - // Extract the future for this branch from the tuple. - let ( $($skip,)* fut, .. ) = &mut futures; - - // Safety: future is stored on the stack above - // and never moved. - let mut fut = unsafe { Pin::new_unchecked(fut) }; - - // Try polling - if fut.as_mut().poll(cx).is_pending() { - is_pending = true; - } else if fut.as_mut().output_mut().expect("expected completed future").is_err() { - return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap())) + if skip == 0 { + if to_run == 0 { + // Every future has been polled + break; + } + to_run -= 1; + + // Extract the future for this branch from the tuple. + let ( $($skip,)* fut, .. ) = &mut *futures; + + // Safety: future is stored on the stack above + // and never moved. + let mut fut = unsafe { Pin::new_unchecked(fut) }; + + // Try polling + if fut.as_mut().poll(cx).is_pending() { + is_pending = true; + } else if fut.as_mut().output_mut().expect("expected completed future").is_err() { + return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap())) + } + } else { + // Future skipped, one less future to skip in the next iteration + skip -= 1; } )* + } if is_pending { Pending @@ -159,13 +204,13 @@ macro_rules! try_join { // ===== Normalize ===== - (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { - $crate::try_join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*) + (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { + $crate::try_join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*) }; // ===== Entry point ===== ( $($e:expr),* $(,)?) => { - $crate::try_join!(@{ () } $($e,)*) + $crate::try_join!(@{ () (0) } $($e,)*) }; } diff --git a/src/net/addr.rs b/src/net/addr.rs index ec4fa19..fb8248f 100644 --- a/src/net/addr.rs +++ b/src/net/addr.rs @@ -1,5 +1,4 @@ -use crate::future; - +use std::future; use std::io; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; @@ -56,7 +55,7 @@ impl sealed::ToSocketAddrsPriv for SocketAddr { fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future { let iter = Some(*self).into_iter(); - future::ok(iter) + future::ready(Ok(iter)) } } @@ -96,7 +95,7 @@ impl sealed::ToSocketAddrsPriv for (IpAddr, u16) { fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future { let iter = Some(SocketAddr::from(*self)).into_iter(); - future::ok(iter) + future::ready(Ok(iter)) } } @@ -137,8 +136,23 @@ impl sealed::ToSocketAddrsPriv for &[SocketAddr] { type Future = ReadyFuture<Self::Iter>; fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future { - let iter = self.to_vec().into_iter(); - future::ok(iter) + #[inline] + fn slice_to_vec(addrs: &[SocketAddr]) -> Vec<SocketAddr> { + addrs.to_vec() + } + + // This uses a helper method because clippy doesn't like the `to_vec()` + // call here (it will allocate, whereas `self.iter().copied()` would + // not), but it's actually necessary in order to ensure that the + // returned iterator is valid for the `'static` lifetime, which the + // borrowed `slice::Iter` iterator would not be. + // + // Note that we can't actually add an `allow` attribute for + // `clippy::unnecessary_to_owned` here, as Tokio's CI runs clippy lints + // on Rust 1.52 to avoid breaking LTS releases of Tokio. Users of newer + // Rust versions who see this lint should just ignore it. + let iter = slice_to_vec(self).into_iter(); + future::ready(Ok(iter)) } } @@ -230,7 +244,7 @@ cfg_net! { type Future = <str as sealed::ToSocketAddrsPriv>::Future; fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future { - (&self[..]).to_socket_addrs(sealed::Internal) + self[..].to_socket_addrs(sealed::Internal) } } } diff --git a/src/net/mod.rs b/src/net/mod.rs index 0b8c1ec..2d317a8 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -23,8 +23,10 @@ //! [`UnixDatagram`]: UnixDatagram mod addr; -#[cfg(feature = "net")] -pub(crate) use addr::to_socket_addrs; +cfg_not_wasi! { + #[cfg(feature = "net")] + pub(crate) use addr::to_socket_addrs; +} pub use addr::ToSocketAddrs; cfg_net! { @@ -33,11 +35,13 @@ cfg_net! { pub mod tcp; pub use tcp::listener::TcpListener; - pub use tcp::socket::TcpSocket; pub use tcp::stream::TcpStream; + cfg_not_wasi! { + pub use tcp::socket::TcpSocket; - mod udp; - pub use udp::UdpSocket; + mod udp; + pub use udp::UdpSocket; + } } cfg_net_unix! { diff --git a/src/net/tcp/listener.rs b/src/net/tcp/listener.rs index 8aecb21..4441313 100644 --- a/src/net/tcp/listener.rs +++ b/src/net/tcp/listener.rs @@ -1,6 +1,9 @@ use crate::io::{Interest, PollEvented}; use crate::net::tcp::TcpStream; -use crate::net::{to_socket_addrs, ToSocketAddrs}; + +cfg_not_wasi! { + use crate::net::{to_socket_addrs, ToSocketAddrs}; +} use std::convert::TryFrom; use std::fmt; @@ -55,68 +58,70 @@ cfg_net! { } impl TcpListener { - /// Creates a new TcpListener, which will be bound to the specified address. - /// - /// The returned listener is ready for accepting connections. - /// - /// Binding with a port number of 0 will request that the OS assigns a port - /// to this listener. The port allocated can be queried via the `local_addr` - /// method. - /// - /// The address type can be any implementor of the [`ToSocketAddrs`] trait. - /// If `addr` yields multiple addresses, bind will be attempted with each of - /// the addresses until one succeeds and returns the listener. If none of - /// the addresses succeed in creating a listener, the error returned from - /// the last attempt (the last address) is returned. - /// - /// This function sets the `SO_REUSEADDR` option on the socket. - /// - /// To configure the socket before binding, you can use the [`TcpSocket`] - /// type. - /// - /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs - /// [`TcpSocket`]: struct@crate::net::TcpSocket - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpListener; - /// - /// use std::io; - /// - /// #[tokio::main] - /// async fn main() -> io::Result<()> { - /// let listener = TcpListener::bind("127.0.0.1:2345").await?; - /// - /// // use the listener - /// - /// # let _ = listener; - /// Ok(()) - /// } - /// ``` - pub async fn bind<A: ToSocketAddrs>(addr: A) -> io::Result<TcpListener> { - let addrs = to_socket_addrs(addr).await?; + cfg_not_wasi! { + /// Creates a new TcpListener, which will be bound to the specified address. + /// + /// The returned listener is ready for accepting connections. + /// + /// Binding with a port number of 0 will request that the OS assigns a port + /// to this listener. The port allocated can be queried via the `local_addr` + /// method. + /// + /// The address type can be any implementor of the [`ToSocketAddrs`] trait. + /// If `addr` yields multiple addresses, bind will be attempted with each of + /// the addresses until one succeeds and returns the listener. If none of + /// the addresses succeed in creating a listener, the error returned from + /// the last attempt (the last address) is returned. + /// + /// This function sets the `SO_REUSEADDR` option on the socket. + /// + /// To configure the socket before binding, you can use the [`TcpSocket`] + /// type. + /// + /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs + /// [`TcpSocket`]: struct@crate::net::TcpSocket + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpListener; + /// + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// let listener = TcpListener::bind("127.0.0.1:2345").await?; + /// + /// // use the listener + /// + /// # let _ = listener; + /// Ok(()) + /// } + /// ``` + pub async fn bind<A: ToSocketAddrs>(addr: A) -> io::Result<TcpListener> { + let addrs = to_socket_addrs(addr).await?; - let mut last_err = None; + let mut last_err = None; - for addr in addrs { - match TcpListener::bind_addr(addr) { - Ok(listener) => return Ok(listener), - Err(e) => last_err = Some(e), + for addr in addrs { + match TcpListener::bind_addr(addr) { + Ok(listener) => return Ok(listener), + Err(e) => last_err = Some(e), + } } - } - Err(last_err.unwrap_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "could not resolve to any address", - ) - })) - } + Err(last_err.unwrap_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "could not resolve to any address", + ) + })) + } - fn bind_addr(addr: SocketAddr) -> io::Result<TcpListener> { - let listener = mio::net::TcpListener::bind(addr)?; - TcpListener::new(listener) + fn bind_addr(addr: SocketAddr) -> io::Result<TcpListener> { + let listener = mio::net::TcpListener::bind(addr)?; + TcpListener::new(listener) + } } /// Accepts a new incoming connection from this listener. @@ -190,15 +195,22 @@ impl TcpListener { /// Creates new `TcpListener` from a `std::net::TcpListener`. /// /// This function is intended to be used to wrap a TCP listener from the - /// standard library in the Tokio equivalent. The conversion assumes nothing - /// about the underlying listener; it is left up to the user to set it in - /// non-blocking mode. + /// standard library in the Tokio equivalent. /// /// This API is typically paired with the `socket2` crate and the `Socket` /// type to build up and customize a listener before it's shipped off to the /// backing event loop. This allows configuration of options like /// `SO_REUSEPORT`, binding to multiple addresses, etc. /// + /// # Notes + /// + /// The caller is responsible for ensuring that the listener is in + /// non-blocking mode. Otherwise all I/O operations on the listener + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::TcpListener::set_nonblocking + /// /// # Examples /// /// ```rust,no_run @@ -216,11 +228,13 @@ impl TcpListener { /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(listener: net::TcpListener) -> io::Result<TcpListener> { let io = mio::net::TcpListener::from_std(listener); let io = PollEvented::new(io)?; @@ -267,11 +281,22 @@ impl TcpListener { .map(|io| io.into_raw_socket()) .map(|raw_socket| unsafe { std::net::TcpListener::from_raw_socket(raw_socket) }) } + + #[cfg(tokio_wasi)] + { + use std::os::wasi::io::{FromRawFd, IntoRawFd}; + self.io + .into_inner() + .map(|io| io.into_raw_fd()) + .map(|raw_fd| unsafe { std::net::TcpListener::from_raw_fd(raw_fd) }) + } } - pub(crate) fn new(listener: mio::net::TcpListener) -> io::Result<TcpListener> { - let io = PollEvented::new(listener)?; - Ok(TcpListener { io }) + cfg_not_wasi! { + pub(crate) fn new(listener: mio::net::TcpListener) -> io::Result<TcpListener> { + let io = PollEvented::new(listener)?; + Ok(TcpListener { io }) + } } /// Returns the local address that this listener is bound to. @@ -384,6 +409,20 @@ mod sys { } } +cfg_unstable! { + #[cfg(tokio_wasi)] + mod sys { + use super::TcpListener; + use std::os::wasi::prelude::*; + + impl AsRawFd for TcpListener { + fn as_raw_fd(&self) -> RawFd { + self.io.as_raw_fd() + } + } + } +} + #[cfg(windows)] mod sys { use super::TcpListener; diff --git a/src/net/tcp/mod.rs b/src/net/tcp/mod.rs index cb8a8b2..734eabe 100644 --- a/src/net/tcp/mod.rs +++ b/src/net/tcp/mod.rs @@ -2,7 +2,9 @@ pub(crate) mod listener; -pub(crate) mod socket; +cfg_not_wasi! { + pub(crate) mod socket; +} mod split; pub use split::{ReadHalf, WriteHalf}; diff --git a/src/net/tcp/socket.rs b/src/net/tcp/socket.rs index f54ff95..09349fe 100644 --- a/src/net/tcp/socket.rs +++ b/src/net/tcp/socket.rs @@ -8,6 +8,7 @@ use std::net::SocketAddr; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; #[cfg(windows)] use std::os::windows::io::{AsRawSocket, FromRawSocket, IntoRawSocket, RawSocket}; +use std::time::Duration; cfg_net! { /// A TCP socket that has not yet been converted to a `TcpStream` or @@ -81,8 +82,9 @@ cfg_net! { /// [`AsRawFd`]: https://doc.rust-lang.org/std/os/unix/io/trait.AsRawFd.html /// [`AsRawSocket`]: https://doc.rust-lang.org/std/os/windows/io/trait.AsRawSocket.html /// [`socket2`]: https://docs.rs/socket2/ + #[cfg_attr(docsrs, doc(alias = "connect_std"))] pub struct TcpSocket { - inner: mio::net::TcpSocket, + inner: socket2::Socket, } } @@ -117,8 +119,7 @@ impl TcpSocket { /// } /// ``` pub fn new_v4() -> io::Result<TcpSocket> { - let inner = mio::net::TcpSocket::new_v4()?; - Ok(TcpSocket { inner }) + TcpSocket::new(socket2::Domain::IPV4) } /// Creates a new socket configured for IPv6. @@ -151,7 +152,34 @@ impl TcpSocket { /// } /// ``` pub fn new_v6() -> io::Result<TcpSocket> { - let inner = mio::net::TcpSocket::new_v6()?; + TcpSocket::new(socket2::Domain::IPV6) + } + + fn new(domain: socket2::Domain) -> io::Result<TcpSocket> { + let ty = socket2::Type::STREAM; + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd" + ))] + let ty = ty.nonblocking(); + let inner = socket2::Socket::new(domain, ty, Some(socket2::Protocol::TCP))?; + #[cfg(not(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd" + )))] + inner.set_nonblocking(true)?; Ok(TcpSocket { inner }) } @@ -182,7 +210,7 @@ impl TcpSocket { /// } /// ``` pub fn set_reuseaddr(&self, reuseaddr: bool) -> io::Result<()> { - self.inner.set_reuseaddr(reuseaddr) + self.inner.set_reuse_address(reuseaddr) } /// Retrieves the value set for `SO_REUSEADDR` on this socket. @@ -208,7 +236,7 @@ impl TcpSocket { /// } /// ``` pub fn reuseaddr(&self) -> io::Result<bool> { - self.inner.get_reuseaddr() + self.inner.reuse_address() } /// Allows the socket to bind to an in-use port. Only available for unix systems @@ -242,7 +270,7 @@ impl TcpSocket { doc(cfg(all(unix, not(target_os = "solaris"), not(target_os = "illumos")))) )] pub fn set_reuseport(&self, reuseport: bool) -> io::Result<()> { - self.inner.set_reuseport(reuseport) + self.inner.set_reuse_port(reuseport) } /// Allows the socket to bind to an in-use port. Only available for unix systems @@ -277,14 +305,14 @@ impl TcpSocket { doc(cfg(all(unix, not(target_os = "solaris"), not(target_os = "illumos")))) )] pub fn reuseport(&self) -> io::Result<bool> { - self.inner.get_reuseport() + self.inner.reuse_port() } /// Sets the size of the TCP send buffer on this socket. /// /// On most operating systems, this sets the `SO_SNDBUF` socket option. pub fn set_send_buffer_size(&self, size: u32) -> io::Result<()> { - self.inner.set_send_buffer_size(size) + self.inner.set_send_buffer_size(size as usize) } /// Returns the size of the TCP send buffer for this socket. @@ -311,14 +339,14 @@ impl TcpSocket { /// /// [`set_send_buffer_size`]: #method.set_send_buffer_size pub fn send_buffer_size(&self) -> io::Result<u32> { - self.inner.get_send_buffer_size() + self.inner.send_buffer_size().map(|n| n as u32) } /// Sets the size of the TCP receive buffer on this socket. /// /// On most operating systems, this sets the `SO_RCVBUF` socket option. pub fn set_recv_buffer_size(&self, size: u32) -> io::Result<()> { - self.inner.set_recv_buffer_size(size) + self.inner.set_recv_buffer_size(size as usize) } /// Returns the size of the TCP receive buffer for this socket. @@ -345,7 +373,112 @@ impl TcpSocket { /// /// [`set_recv_buffer_size`]: #method.set_recv_buffer_size pub fn recv_buffer_size(&self) -> io::Result<u32> { - self.inner.get_recv_buffer_size() + self.inner.recv_buffer_size().map(|n| n as u32) + } + + /// Sets the linger duration of this socket by setting the SO_LINGER option. + /// + /// This option controls the action taken when a stream has unsent messages and the stream is + /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the + /// data or until the time expires. + /// + /// If SO_LINGER is not specified, and the socket is closed, the system handles the call in a + /// way that allows the process to continue as quickly as possible. + pub fn set_linger(&self, dur: Option<Duration>) -> io::Result<()> { + self.inner.set_linger(dur) + } + + /// Reads the linger duration for this socket by getting the `SO_LINGER` + /// option. + /// + /// For more information about this option, see [`set_linger`]. + /// + /// [`set_linger`]: TcpSocket::set_linger + pub fn linger(&self) -> io::Result<Option<Duration>> { + self.inner.linger() + } + + /// Gets the value of the `IP_TOS` option for this socket. + /// + /// For more information about this option, see [`set_tos`]. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + /// + /// [`set_tos`]: Self::set_tos + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn tos(&self) -> io::Result<u32> { + self.inner.tos() + } + + /// Sets the value for the `IP_TOS` option on this socket. + /// + /// This value sets the type-of-service field that is used in every packet + /// sent from this socket. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn set_tos(&self, tos: u32) -> io::Result<()> { + self.inner.set_tos(tos) + } + + /// Gets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// This value gets the socket binded device's interface name. + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))] + #[cfg_attr( + docsrs, + doc(cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))) + )] + pub fn device(&self) -> io::Result<Option<Vec<u8>>> { + self.inner.device() + } + + /// Sets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// If a socket is bound to an interface, only packets received from that + /// particular interface are processed by the socket. Note that this only + /// works for some socket types, particularly `AF_INET` sockets. + /// + /// If `interface` is `None` or an empty string it removes the binding. + #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] + #[cfg_attr( + docsrs, + doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))) + )] + pub fn bind_device(&self, interface: Option<&[u8]>) -> io::Result<()> { + self.inner.bind_device(interface) } /// Gets the local address of this socket. @@ -371,7 +504,12 @@ impl TcpSocket { /// } /// ``` pub fn local_addr(&self) -> io::Result<SocketAddr> { - self.inner.get_localaddr() + self.inner.local_addr().and_then(convert_address) + } + + /// Returns the value of the `SO_ERROR` option. + pub fn take_error(&self) -> io::Result<Option<io::Error>> { + self.inner.take_error() } /// Binds the socket to the given address. @@ -403,7 +541,7 @@ impl TcpSocket { /// } /// ``` pub fn bind(&self, addr: SocketAddr) -> io::Result<()> { - self.inner.bind(addr) + self.inner.bind(&addr.into()) } /// Establishes a TCP connection with a peer at the specified socket address. @@ -439,7 +577,32 @@ impl TcpSocket { /// } /// ``` pub async fn connect(self, addr: SocketAddr) -> io::Result<TcpStream> { - let mio = self.inner.connect(addr)?; + if let Err(err) = self.inner.connect(&addr.into()) { + #[cfg(unix)] + if err.raw_os_error() != Some(libc::EINPROGRESS) { + return Err(err); + } + #[cfg(windows)] + if err.kind() != io::ErrorKind::WouldBlock { + return Err(err); + } + } + #[cfg(unix)] + let mio = { + use std::os::unix::io::{FromRawFd, IntoRawFd}; + + let raw_fd = self.inner.into_raw_fd(); + unsafe { mio::net::TcpStream::from_raw_fd(raw_fd) } + }; + + #[cfg(windows)] + let mio = { + use std::os::windows::io::{FromRawSocket, IntoRawSocket}; + + let raw_socket = self.inner.into_raw_socket(); + unsafe { mio::net::TcpStream::from_raw_socket(raw_socket) } + }; + TcpStream::connect_mio(mio).await } @@ -479,7 +642,23 @@ impl TcpSocket { /// } /// ``` pub fn listen(self, backlog: u32) -> io::Result<TcpListener> { - let mio = self.inner.listen(backlog)?; + self.inner.listen(backlog as i32)?; + #[cfg(unix)] + let mio = { + use std::os::unix::io::{FromRawFd, IntoRawFd}; + + let raw_fd = self.inner.into_raw_fd(); + unsafe { mio::net::TcpListener::from_raw_fd(raw_fd) } + }; + + #[cfg(windows)] + let mio = { + use std::os::windows::io::{FromRawSocket, IntoRawSocket}; + + let raw_socket = self.inner.into_raw_socket(); + unsafe { mio::net::TcpListener::from_raw_socket(raw_socket) } + }; + TcpListener::new(mio) } @@ -491,6 +670,15 @@ impl TcpSocket { /// [`std::net::TcpStream`]: struct@std::net::TcpStream /// [`socket2`]: https://docs.rs/socket2/ /// + /// # Notes + /// + /// The caller is responsible for ensuring that the socket is in + /// non-blocking mode. Otherwise all I/O operations on the socket + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::TcpStream::set_nonblocking + /// /// # Examples /// /// ``` @@ -499,8 +687,8 @@ impl TcpSocket { /// /// #[tokio::main] /// async fn main() -> std::io::Result<()> { - /// /// let socket2_socket = Socket::new(Domain::IPV4, Type::STREAM, None)?; + /// socket2_socket.set_nonblocking(true)?; /// /// let socket = TcpSocket::from_std_stream(socket2_socket.into()); /// @@ -526,6 +714,16 @@ impl TcpSocket { } } +fn convert_address(address: socket2::SockAddr) -> io::Result<SocketAddr> { + match address.as_socket() { + Some(address) => Ok(address), + None => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid address family (not IPv4 or IPv6)", + )), + } +} + impl fmt::Debug for TcpSocket { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { self.inner.fmt(fmt) @@ -548,7 +746,7 @@ impl FromRawFd for TcpSocket { /// The caller is responsible for ensuring that the socket is in /// non-blocking mode. unsafe fn from_raw_fd(fd: RawFd) -> TcpSocket { - let inner = mio::net::TcpSocket::from_raw_fd(fd); + let inner = socket2::Socket::from_raw_fd(fd); TcpSocket { inner } } } @@ -583,7 +781,7 @@ impl FromRawSocket for TcpSocket { /// The caller is responsible for ensuring that the socket is in /// non-blocking mode. unsafe fn from_raw_socket(socket: RawSocket) -> TcpSocket { - let inner = mio::net::TcpSocket::from_raw_socket(socket); + let inner = socket2::Socket::from_raw_socket(socket); TcpSocket { inner } } } diff --git a/src/net/tcp/split.rs b/src/net/tcp/split.rs index 0e02928..343d4fd 100644 --- a/src/net/tcp/split.rs +++ b/src/net/tcp/split.rs @@ -145,6 +145,12 @@ impl ReadHalf<'_> { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// This function is equivalent to [`TcpStream::ready`]. /// /// # Cancel safety @@ -190,8 +196,12 @@ impl ReadHalf<'_> { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> { self.0.try_read(buf) @@ -269,6 +279,12 @@ impl WriteHalf<'_> { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// This function is equivalent to [`TcpStream::ready`]. /// /// # Cancel safety diff --git a/src/net/tcp/split_owned.rs b/src/net/tcp/split_owned.rs index ef4e7b5..53fc5f0 100644 --- a/src/net/tcp/split_owned.rs +++ b/src/net/tcp/split_owned.rs @@ -200,6 +200,12 @@ impl OwnedReadHalf { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// This function is equivalent to [`TcpStream::ready`]. /// /// # Cancel safety @@ -245,8 +251,12 @@ impl OwnedReadHalf { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> { self.inner.try_read(buf) @@ -351,6 +361,12 @@ impl OwnedWriteHalf { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// This function is equivalent to [`TcpStream::ready`]. /// /// # Cancel safety @@ -474,12 +490,12 @@ impl AsyncWrite for OwnedWriteHalf { impl AsRef<TcpStream> for OwnedReadHalf { fn as_ref(&self) -> &TcpStream { - &*self.inner + &self.inner } } impl AsRef<TcpStream> for OwnedWriteHalf { fn as_ref(&self) -> &TcpStream { - &*self.inner + &self.inner } } diff --git a/src/net/tcp/stream.rs b/src/net/tcp/stream.rs index 60d20fd..b17d33f 100644 --- a/src/net/tcp/stream.rs +++ b/src/net/tcp/stream.rs @@ -1,8 +1,12 @@ -use crate::future::poll_fn; +cfg_not_wasi! { + use crate::future::poll_fn; + use crate::net::{to_socket_addrs, ToSocketAddrs}; + use std::time::Duration; +} + use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready}; use crate::net::tcp::split::{split, ReadHalf, WriteHalf}; use crate::net::tcp::split_owned::{split_owned, OwnedReadHalf, OwnedWriteHalf}; -use crate::net::{to_socket_addrs, ToSocketAddrs}; use std::convert::TryFrom; use std::fmt; @@ -10,7 +14,6 @@ use std::io; use std::net::{Shutdown, SocketAddr}; use std::pin::Pin; use std::task::{Context, Poll}; -use std::time::Duration; cfg_io_util! { use bytes::BufMut; @@ -70,86 +73,88 @@ cfg_net! { } impl TcpStream { - /// Opens a TCP connection to a remote host. - /// - /// `addr` is an address of the remote host. Anything which implements the - /// [`ToSocketAddrs`] trait can be supplied as the address. If `addr` - /// yields multiple addresses, connect will be attempted with each of the - /// addresses until a connection is successful. If none of the addresses - /// result in a successful connection, the error returned from the last - /// connection attempt (the last address) is returned. - /// - /// To configure the socket before connecting, you can use the [`TcpSocket`] - /// type. - /// - /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs - /// [`TcpSocket`]: struct@crate::net::TcpSocket - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpStream; - /// use tokio::io::AsyncWriteExt; - /// use std::error::Error; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box<dyn Error>> { - /// // Connect to a peer - /// let mut stream = TcpStream::connect("127.0.0.1:8080").await?; - /// - /// // Write some data. - /// stream.write_all(b"hello world!").await?; - /// - /// Ok(()) - /// } - /// ``` - /// - /// The [`write_all`] method is defined on the [`AsyncWriteExt`] trait. - /// - /// [`write_all`]: fn@crate::io::AsyncWriteExt::write_all - /// [`AsyncWriteExt`]: trait@crate::io::AsyncWriteExt - pub async fn connect<A: ToSocketAddrs>(addr: A) -> io::Result<TcpStream> { - let addrs = to_socket_addrs(addr).await?; + cfg_not_wasi! { + /// Opens a TCP connection to a remote host. + /// + /// `addr` is an address of the remote host. Anything which implements the + /// [`ToSocketAddrs`] trait can be supplied as the address. If `addr` + /// yields multiple addresses, connect will be attempted with each of the + /// addresses until a connection is successful. If none of the addresses + /// result in a successful connection, the error returned from the last + /// connection attempt (the last address) is returned. + /// + /// To configure the socket before connecting, you can use the [`TcpSocket`] + /// type. + /// + /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs + /// [`TcpSocket`]: struct@crate::net::TcpSocket + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpStream; + /// use tokio::io::AsyncWriteExt; + /// use std::error::Error; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box<dyn Error>> { + /// // Connect to a peer + /// let mut stream = TcpStream::connect("127.0.0.1:8080").await?; + /// + /// // Write some data. + /// stream.write_all(b"hello world!").await?; + /// + /// Ok(()) + /// } + /// ``` + /// + /// The [`write_all`] method is defined on the [`AsyncWriteExt`] trait. + /// + /// [`write_all`]: fn@crate::io::AsyncWriteExt::write_all + /// [`AsyncWriteExt`]: trait@crate::io::AsyncWriteExt + pub async fn connect<A: ToSocketAddrs>(addr: A) -> io::Result<TcpStream> { + let addrs = to_socket_addrs(addr).await?; - let mut last_err = None; + let mut last_err = None; - for addr in addrs { - match TcpStream::connect_addr(addr).await { - Ok(stream) => return Ok(stream), - Err(e) => last_err = Some(e), + for addr in addrs { + match TcpStream::connect_addr(addr).await { + Ok(stream) => return Ok(stream), + Err(e) => last_err = Some(e), + } } + + Err(last_err.unwrap_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "could not resolve to any address", + ) + })) } - Err(last_err.unwrap_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "could not resolve to any address", - ) - })) - } + /// Establishes a connection to the specified `addr`. + async fn connect_addr(addr: SocketAddr) -> io::Result<TcpStream> { + let sys = mio::net::TcpStream::connect(addr)?; + TcpStream::connect_mio(sys).await + } - /// Establishes a connection to the specified `addr`. - async fn connect_addr(addr: SocketAddr) -> io::Result<TcpStream> { - let sys = mio::net::TcpStream::connect(addr)?; - TcpStream::connect_mio(sys).await - } + pub(crate) async fn connect_mio(sys: mio::net::TcpStream) -> io::Result<TcpStream> { + let stream = TcpStream::new(sys)?; - pub(crate) async fn connect_mio(sys: mio::net::TcpStream) -> io::Result<TcpStream> { - let stream = TcpStream::new(sys)?; + // Once we've connected, wait for the stream to be writable as + // that's when the actual connection has been initiated. Once we're + // writable we check for `take_socket_error` to see if the connect + // actually hit an error or not. + // + // If all that succeeded then we ship everything on up. + poll_fn(|cx| stream.io.registration().poll_write_ready(cx)).await?; - // Once we've connected, wait for the stream to be writable as - // that's when the actual connection has been initiated. Once we're - // writable we check for `take_socket_error` to see if the connect - // actually hit an error or not. - // - // If all that succeeded then we ship everything on up. - poll_fn(|cx| stream.io.registration().poll_write_ready(cx)).await?; + if let Some(e) = stream.io.take_error()? { + return Err(e); + } - if let Some(e) = stream.io.take_error()? { - return Err(e); + Ok(stream) } - - Ok(stream) } pub(crate) fn new(connected: mio::net::TcpStream) -> io::Result<TcpStream> { @@ -160,9 +165,16 @@ impl TcpStream { /// Creates new `TcpStream` from a `std::net::TcpStream`. /// /// This function is intended to be used to wrap a TCP stream from the - /// standard library in the Tokio equivalent. The conversion assumes nothing - /// about the underlying stream; it is left up to the user to set it in - /// non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the stream is in + /// non-blocking mode. Otherwise all I/O operations on the stream + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::TcpStream::set_nonblocking /// /// # Examples /// @@ -181,11 +193,13 @@ impl TcpStream { /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(stream: std::net::TcpStream) -> io::Result<TcpStream> { let io = mio::net::TcpStream::from_std(stream); let io = PollEvented::new(io)?; @@ -244,6 +258,15 @@ impl TcpStream { .map(|io| io.into_raw_socket()) .map(|raw_socket| unsafe { std::net::TcpStream::from_raw_socket(raw_socket) }) } + + #[cfg(tokio_wasi)] + { + use std::os::wasi::io::{FromRawFd, IntoRawFd}; + self.io + .into_inner() + .map(|io| io.into_raw_fd()) + .map(|raw_fd| unsafe { std::net::TcpStream::from_raw_fd(raw_fd) }) + } } /// Returns the local address that this stream is bound to. @@ -264,6 +287,11 @@ impl TcpStream { self.io.local_addr() } + /// Returns the value of the `SO_ERROR` option. + pub fn take_error(&self) -> io::Result<Option<io::Error>> { + self.io.take_error() + } + /// Returns the remote address that this stream is connected to. /// /// # Examples @@ -356,6 +384,12 @@ impl TcpStream { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -387,7 +421,7 @@ impl TcpStream { /// // if the readiness event is a false positive. /// match stream.try_read(&mut data) { /// Ok(n) => { - /// println!("read {} bytes", n); + /// println!("read {} bytes", n); /// } /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { /// continue; @@ -526,8 +560,12 @@ impl TcpStream { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -939,7 +977,7 @@ impl TcpStream { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -958,6 +996,11 @@ impl TcpStream { /// defined on the Tokio `TcpStream` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: TcpStream::readable() @@ -968,7 +1011,9 @@ impl TcpStream { interest: Interest, f: impl FnOnce() -> io::Result<R>, ) -> io::Result<R> { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) } /// Receives data on the socket from the remote address to which it is @@ -1070,69 +1115,53 @@ impl TcpStream { self.io.set_nodelay(nodelay) } - /// Reads the linger duration for this socket by getting the `SO_LINGER` - /// option. - /// - /// For more information about this option, see [`set_linger`]. - /// - /// [`set_linger`]: TcpStream::set_linger - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpStream; - /// - /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> { - /// let stream = TcpStream::connect("127.0.0.1:8080").await?; - /// - /// println!("{:?}", stream.linger()?); - /// # Ok(()) - /// # } - /// ``` - pub fn linger(&self) -> io::Result<Option<Duration>> { - let mio_socket = std::mem::ManuallyDrop::new(self.to_mio()); - - mio_socket.get_linger() - } - - /// Sets the linger duration of this socket by setting the SO_LINGER option. - /// - /// This option controls the action taken when a stream has unsent messages and the stream is - /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the - /// data or until the time expires. - /// - /// If SO_LINGER is not specified, and the stream is closed, the system handles the call in a - /// way that allows the process to continue as quickly as possible. - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpStream; - /// - /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> { - /// let stream = TcpStream::connect("127.0.0.1:8080").await?; - /// - /// stream.set_linger(None)?; - /// # Ok(()) - /// # } - /// ``` - pub fn set_linger(&self, dur: Option<Duration>) -> io::Result<()> { - let mio_socket = std::mem::ManuallyDrop::new(self.to_mio()); - - mio_socket.set_linger(dur) - } - - fn to_mio(&self) -> mio::net::TcpSocket { - #[cfg(windows)] - { - use std::os::windows::io::{AsRawSocket, FromRawSocket}; - unsafe { mio::net::TcpSocket::from_raw_socket(self.as_raw_socket()) } + cfg_not_wasi! { + /// Reads the linger duration for this socket by getting the `SO_LINGER` + /// option. + /// + /// For more information about this option, see [`set_linger`]. + /// + /// [`set_linger`]: TcpStream::set_linger + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpStream; + /// + /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> { + /// let stream = TcpStream::connect("127.0.0.1:8080").await?; + /// + /// println!("{:?}", stream.linger()?); + /// # Ok(()) + /// # } + /// ``` + pub fn linger(&self) -> io::Result<Option<Duration>> { + socket2::SockRef::from(self).linger() } - #[cfg(unix)] - { - use std::os::unix::io::{AsRawFd, FromRawFd}; - unsafe { mio::net::TcpSocket::from_raw_fd(self.as_raw_fd()) } + /// Sets the linger duration of this socket by setting the SO_LINGER option. + /// + /// This option controls the action taken when a stream has unsent messages and the stream is + /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the + /// data or until the time expires. + /// + /// If SO_LINGER is not specified, and the stream is closed, the system handles the call in a + /// way that allows the process to continue as quickly as possible. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpStream; + /// + /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> { + /// let stream = TcpStream::connect("127.0.0.1:8080").await?; + /// + /// stream.set_linger(None)?; + /// # Ok(()) + /// # } + /// ``` + pub fn set_linger(&self, dur: Option<Duration>) -> io::Result<()> { + socket2::SockRef::from(self).set_linger(dur) } } @@ -1326,3 +1355,15 @@ mod sys { } } } + +#[cfg(all(tokio_unstable, tokio_wasi))] +mod sys { + use super::TcpStream; + use std::os::wasi::prelude::*; + + impl AsRawFd for TcpStream { + fn as_raw_fd(&self) -> RawFd { + self.io.as_raw_fd() + } + } +} diff --git a/src/net/udp.rs b/src/net/udp.rs index 504d74e..213d914 100644 --- a/src/net/udp.rs +++ b/src/net/udp.rs @@ -128,6 +128,10 @@ impl UdpSocket { /// This function will create a new UDP socket and attempt to bind it to /// the `addr` provided. /// + /// Binding with a port number of 0 will request that the OS assigns a port + /// to this listener. The port allocated can be queried via the `local_addr` + /// method. + /// /// # Example /// /// ```no_run @@ -166,6 +170,7 @@ impl UdpSocket { UdpSocket::new(sys) } + #[track_caller] fn new(socket: mio::net::UdpSocket) -> io::Result<UdpSocket> { let io = PollEvented::new(socket)?; Ok(UdpSocket { io }) @@ -174,14 +179,21 @@ impl UdpSocket { /// Creates new `UdpSocket` from a previously bound `std::net::UdpSocket`. /// /// This function is intended to be used to wrap a UDP socket from the - /// standard library in the Tokio equivalent. The conversion assumes nothing - /// about the underlying socket; it is left up to the user to set it in - /// non-blocking mode. + /// standard library in the Tokio equivalent. /// /// This can be used in conjunction with socket2's `Socket` interface to /// configure a socket before it's handed off, such as setting options like /// `reuse_address` or binding to multiple addresses. /// + /// # Notes + /// + /// The caller is responsible for ensuring that the socket is in + /// non-blocking mode. Otherwise all I/O operations on the socket + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::UdpSocket::set_nonblocking + /// /// # Panics /// /// This function panics if thread-local runtime is not set. @@ -206,6 +218,7 @@ impl UdpSocket { /// # Ok(()) /// # } /// ``` + #[track_caller] pub fn from_std(socket: net::UdpSocket) -> io::Result<UdpSocket> { let io = mio::net::UdpSocket::from_std(socket); UdpSocket::new(io) @@ -253,6 +266,10 @@ impl UdpSocket { } } + fn as_socket(&self) -> socket2::SockRef<'_> { + socket2::SockRef::from(self) + } + /// Returns the local address that this socket is bound to. /// /// # Example @@ -274,6 +291,28 @@ impl UdpSocket { self.io.local_addr() } + /// Returns the socket address of the remote peer this socket was connected to. + /// + /// # Example + /// + /// ``` + /// use tokio::net::UdpSocket; + /// + /// # use std::{io, net::SocketAddr}; + /// # #[tokio::main] + /// # async fn main() -> io::Result<()> { + /// let addr = "0.0.0.0:8080".parse::<SocketAddr>().unwrap(); + /// let peer = "127.0.0.1:11100".parse::<SocketAddr>().unwrap(); + /// let sock = UdpSocket::bind(addr).await?; + /// sock.connect(peer).await?; + /// assert_eq!(peer, sock.peer_addr()?); + /// # Ok(()) + /// # } + /// ``` + pub fn peer_addr(&self) -> io::Result<SocketAddr> { + self.io.peer_addr() + } + /// Connects the UDP socket setting the default destination for send() and /// limiting packets that are read via recv from the address specified in /// `addr`. @@ -325,7 +364,9 @@ impl UdpSocket { /// /// The function may complete without the socket being ready. This is a /// false-positive and attempting an operation will return with - /// `io::ErrorKind::WouldBlock`. + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// # Cancel safety /// @@ -708,7 +749,7 @@ impl UdpSocket { /// /// # Cancel safety /// - /// This method is cancel safe. If `recv_from` is used as the event in a + /// This method is cancel safe. If `recv` is used as the event in a /// [`tokio::select!`](crate::select) statement and some other branch /// completes first, it is guaranteed that no messages were received on this /// socket. @@ -893,7 +934,7 @@ impl UdpSocket { // Safety: We trust `UdpSocket::recv` to have filled up `n` bytes in the // buffer. - let n = (&*self.io).recv(dst)?; + let n = (*self.io).recv(dst)?; unsafe { buf.advance_mut(n); @@ -957,7 +998,7 @@ impl UdpSocket { // Safety: We trust `UdpSocket::recv_from` to have filled up `n` bytes in the // buffer. - let (n, addr) = (&*self.io).recv_from(dst)?; + let (n, addr) = (*self.io).recv_from(dst)?; unsafe { buf.advance_mut(n); @@ -1239,7 +1280,7 @@ impl UdpSocket { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -1258,6 +1299,11 @@ impl UdpSocket { /// defined on the Tokio `UdpSocket` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: UdpSocket::readable() @@ -1268,7 +1314,9 @@ impl UdpSocket { interest: Interest, f: impl FnOnce() -> io::Result<R>, ) -> io::Result<R> { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) } /// Receives data from the socket, without removing it from the input queue. @@ -1480,6 +1528,89 @@ impl UdpSocket { self.io.set_ttl(ttl) } + /// Gets the value of the `IP_TOS` option for this socket. + /// + /// For more information about this option, see [`set_tos`]. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + /// + /// [`set_tos`]: Self::set_tos + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn tos(&self) -> io::Result<u32> { + self.as_socket().tos() + } + + /// Sets the value for the `IP_TOS` option on this socket. + /// + /// This value sets the type-of-service field that is used in every packet + /// sent from this socket. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn set_tos(&self, tos: u32) -> io::Result<()> { + self.as_socket().set_tos(tos) + } + + /// Gets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// This value gets the socket-bound device's interface name. + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))] + #[cfg_attr( + docsrs, + doc(cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))) + )] + pub fn device(&self) -> io::Result<Option<Vec<u8>>> { + self.as_socket().device() + } + + /// Sets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// If a socket is bound to an interface, only packets received from that + /// particular interface are processed by the socket. Note that this only + /// works for some socket types, particularly `AF_INET` sockets. + /// + /// If `interface` is `None` or an empty string it removes the binding. + #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] + #[cfg_attr( + docsrs, + doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))) + )] + pub fn bind_device(&self, interface: Option<&[u8]>) -> io::Result<()> { + self.as_socket().bind_device(interface) + } + /// Executes an operation of the `IP_ADD_MEMBERSHIP` type. /// /// This function specifies a new multicast group for this socket to join. diff --git a/src/net/unix/datagram/socket.rs b/src/net/unix/datagram/socket.rs index d5b6186..76c6b19 100644 --- a/src/net/unix/datagram/socket.rs +++ b/src/net/unix/datagram/socket.rs @@ -104,7 +104,9 @@ impl UnixDatagram { /// /// The function may complete without the socket being ready. This is a /// false-positive and attempting an operation will return with - /// `io::ErrorKind::WouldBlock`. + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// # Cancel safety /// @@ -424,13 +426,21 @@ impl UnixDatagram { /// Creates new `UnixDatagram` from a `std::os::unix::net::UnixDatagram`. /// /// This function is intended to be used to wrap a UnixDatagram from the - /// standard library in the Tokio equivalent. The conversion assumes - /// nothing about the underlying datagram; it is left up to the user to set - /// it in non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the socker is in + /// non-blocking mode. Otherwise all I/O operations on the socket + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::os::unix::net::UnixDatagram::set_nonblocking /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a Tokio runtime, otherwise runtime can be set @@ -457,6 +467,7 @@ impl UnixDatagram { /// # Ok(()) /// # } /// ``` + #[track_caller] pub fn from_std(datagram: net::UnixDatagram) -> io::Result<UnixDatagram> { let socket = mio::net::UnixDatagram::from_std(datagram); let io = PollEvented::new(socket)?; @@ -466,21 +477,19 @@ impl UnixDatagram { /// Turns a [`tokio::net::UnixDatagram`] into a [`std::os::unix::net::UnixDatagram`]. /// /// The returned [`std::os::unix::net::UnixDatagram`] will have nonblocking - /// mode set as `true`. Use [`set_nonblocking`] to change the blocking mode + /// mode set as `true`. Use [`set_nonblocking`] to change the blocking mode /// if needed. /// /// # Examples /// /// ```rust,no_run - /// use std::error::Error; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box<dyn Error>> { - /// let tokio_socket = tokio::net::UnixDatagram::bind("127.0.0.1:0")?; - /// let std_socket = tokio_socket.into_std()?; - /// std_socket.set_nonblocking(false)?; - /// Ok(()) - /// } + /// # use std::error::Error; + /// # async fn dox() -> Result<(), Box<dyn Error>> { + /// let tokio_socket = tokio::net::UnixDatagram::bind("/path/to/the/socket")?; + /// let std_socket = tokio_socket.into_std()?; + /// std_socket.set_nonblocking(false)?; + /// # Ok(()) + /// # } /// ``` /// /// [`tokio::net::UnixDatagram`]: UnixDatagram @@ -844,7 +853,7 @@ impl UnixDatagram { // Safety: We trust `UnixDatagram::recv_from` to have filled up `n` bytes in the // buffer. - let (n, addr) = (&*self.io).recv_from(dst)?; + let (n, addr) = (*self.io).recv_from(dst)?; unsafe { buf.advance_mut(n); @@ -907,7 +916,7 @@ impl UnixDatagram { // Safety: We trust `UnixDatagram::recv` to have filled up `n` bytes in the // buffer. - let n = (&*self.io).recv(dst)?; + let n = (*self.io).recv(dst)?; unsafe { buf.advance_mut(n); @@ -1212,7 +1221,7 @@ impl UnixDatagram { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -1231,6 +1240,11 @@ impl UnixDatagram { /// defined on the Tokio `UnixDatagram` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: UnixDatagram::readable() @@ -1241,7 +1255,9 @@ impl UnixDatagram { interest: Interest, f: impl FnOnce() -> io::Result<R>, ) -> io::Result<R> { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) } /// Returns the local address that this socket is bound to. diff --git a/src/net/unix/listener.rs b/src/net/unix/listener.rs index 1785f8b..9887f73 100644 --- a/src/net/unix/listener.rs +++ b/src/net/unix/listener.rs @@ -54,11 +54,13 @@ impl UnixListener { /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn bind<P>(path: P) -> io::Result<UnixListener> where P: AsRef<Path>, @@ -71,17 +73,41 @@ impl UnixListener { /// Creates new `UnixListener` from a `std::os::unix::net::UnixListener `. /// /// This function is intended to be used to wrap a UnixListener from the - /// standard library in the Tokio equivalent. The conversion assumes - /// nothing about the underlying listener; it is left up to the user to set - /// it in non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the listener is in + /// non-blocking mode. Otherwise all I/O operations on the listener + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::os::unix::net::UnixListener::set_nonblocking + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::UnixListener; + /// use std::os::unix::net::UnixListener as StdUnixListener; + /// # use std::error::Error; + /// + /// # async fn dox() -> Result<(), Box<dyn Error>> { + /// let std_listener = StdUnixListener::bind("/path/to/the/socket")?; + /// std_listener.set_nonblocking(true)?; + /// let listener = UnixListener::from_std(std_listener)?; + /// # Ok(()) + /// # } + /// ``` /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(listener: net::UnixListener) -> io::Result<UnixListener> { let listener = mio::net::UnixListener::from_std(listener); let io = PollEvented::new(listener)?; @@ -91,20 +117,18 @@ impl UnixListener { /// Turns a [`tokio::net::UnixListener`] into a [`std::os::unix::net::UnixListener`]. /// /// The returned [`std::os::unix::net::UnixListener`] will have nonblocking mode - /// set as `true`. Use [`set_nonblocking`] to change the blocking mode if needed. + /// set as `true`. Use [`set_nonblocking`] to change the blocking mode if needed. /// /// # Examples /// /// ```rust,no_run - /// use std::error::Error; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box<dyn Error>> { - /// let tokio_listener = tokio::net::UnixListener::bind("127.0.0.1:0")?; - /// let std_listener = tokio_listener.into_std()?; - /// std_listener.set_nonblocking(false)?; - /// Ok(()) - /// } + /// # use std::error::Error; + /// # async fn dox() -> Result<(), Box<dyn Error>> { + /// let tokio_listener = tokio::net::UnixListener::bind("/path/to/the/socket")?; + /// let std_listener = tokio_listener.into_std()?; + /// std_listener.set_nonblocking(false)?; + /// # Ok(()) + /// # } /// ``` /// /// [`tokio::net::UnixListener`]: UnixListener diff --git a/src/net/unix/mod.rs b/src/net/unix/mod.rs index 14cb456..97b6327 100644 --- a/src/net/unix/mod.rs +++ b/src/net/unix/mod.rs @@ -1,5 +1,4 @@ //! Unix domain socket utility types. - // This module does not currently provide any public API, but it was // unintentionally defined as a public module. Hide it from the documentation // instead of changing it to a private module to avoid breakage. @@ -22,3 +21,15 @@ pub(crate) use stream::UnixStream; mod ucred; pub use ucred::UCred; + +/// A type representing process and process group IDs. +#[allow(non_camel_case_types)] +pub type uid_t = u32; + +/// A type representing user ID. +#[allow(non_camel_case_types)] +pub type gid_t = u32; + +/// A type representing group ID. +#[allow(non_camel_case_types)] +pub type pid_t = i32; diff --git a/src/net/unix/split.rs b/src/net/unix/split.rs index d4686c2..816a257 100644 --- a/src/net/unix/split.rs +++ b/src/net/unix/split.rs @@ -100,8 +100,12 @@ impl ReadHalf<'_> { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> { self.0.try_read(buf) @@ -178,6 +182,12 @@ impl WriteHalf<'_> { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method diff --git a/src/net/unix/split_owned.rs b/src/net/unix/split_owned.rs index 9c3a2a4..2cb561d 100644 --- a/src/net/unix/split_owned.rs +++ b/src/net/unix/split_owned.rs @@ -114,6 +114,12 @@ impl OwnedReadHalf { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -155,8 +161,12 @@ impl OwnedReadHalf { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> { self.inner.try_read(buf) @@ -261,6 +271,12 @@ impl OwnedWriteHalf { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -382,12 +398,12 @@ impl AsyncWrite for OwnedWriteHalf { impl AsRef<UnixStream> for OwnedReadHalf { fn as_ref(&self) -> &UnixStream { - &*self.inner + &self.inner } } impl AsRef<UnixStream> for OwnedWriteHalf { fn as_ref(&self) -> &UnixStream { - &*self.inner + &self.inner } } diff --git a/src/net/unix/stream.rs b/src/net/unix/stream.rs index 4e7ef87..c249bf4 100644 --- a/src/net/unix/stream.rs +++ b/src/net/unix/stream.rs @@ -22,8 +22,8 @@ cfg_io_util! { cfg_net_unix! { /// A structure representing a connected Unix socket. /// - /// This socket can be connected directly with `UnixStream::connect` or accepted - /// from a listener with `UnixListener::incoming`. Additionally, a pair of + /// This socket can be connected directly with [`UnixStream::connect`] or accepted + /// from a listener with [`UnixListener::accept`]. Additionally, a pair of /// anonymous Unix sockets can be created with `UnixStream::pair`. /// /// To shut down the stream in the write direction, you can call the @@ -32,6 +32,7 @@ cfg_net_unix! { /// the stream in one direction. /// /// [`shutdown()`]: fn@crate::io::AsyncWriteExt::shutdown + /// [`UnixListener::accept`]: crate::net::UnixListener::accept pub struct UnixStream { io: PollEvented<mio::net::UnixStream>, } @@ -65,6 +66,12 @@ impl UnixStream { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -239,8 +246,12 @@ impl UnixStream { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -656,7 +667,7 @@ impl UnixStream { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -675,6 +686,11 @@ impl UnixStream { /// defined on the Tokio `UnixStream` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: UnixStream::readable() @@ -685,23 +701,49 @@ impl UnixStream { interest: Interest, f: impl FnOnce() -> io::Result<R>, ) -> io::Result<R> { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) } /// Creates new `UnixStream` from a `std::os::unix::net::UnixStream`. /// /// This function is intended to be used to wrap a UnixStream from the - /// standard library in the Tokio equivalent. The conversion assumes - /// nothing about the underlying stream; it is left up to the user to set - /// it in non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the stream is in + /// non-blocking mode. Otherwise all I/O operations on the stream + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::os::unix::net::UnixStream::set_nonblocking + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::UnixStream; + /// use std::os::unix::net::UnixStream as StdUnixStream; + /// # use std::error::Error; + /// + /// # async fn dox() -> Result<(), Box<dyn Error>> { + /// let std_stream = StdUnixStream::connect("/path/to/the/socket")?; + /// std_stream.set_nonblocking(true)?; + /// let stream = UnixStream::from_std(std_stream)?; + /// # Ok(()) + /// # } + /// ``` /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(stream: net::UnixStream) -> io::Result<UnixStream> { let stream = mio::net::UnixStream::from_std(stream); let io = PollEvented::new(stream)?; diff --git a/src/net/unix/ucred.rs b/src/net/unix/ucred.rs index 865303b..3cb61d7 100644 --- a/src/net/unix/ucred.rs +++ b/src/net/unix/ucred.rs @@ -1,24 +1,24 @@ -use libc::{gid_t, pid_t, uid_t}; +use crate::net::unix; /// Credentials of a process. #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] pub struct UCred { /// PID (process ID) of the process. - pid: Option<pid_t>, + pid: Option<unix::pid_t>, /// UID (user ID) of the process. - uid: uid_t, + uid: unix::uid_t, /// GID (group ID) of the process. - gid: gid_t, + gid: unix::gid_t, } impl UCred { /// Gets UID (user ID) of the process. - pub fn uid(&self) -> uid_t { + pub fn uid(&self) -> unix::uid_t { self.uid } /// Gets GID (group ID) of the process. - pub fn gid(&self) -> gid_t { + pub fn gid(&self) -> unix::gid_t { self.gid } @@ -26,7 +26,7 @@ impl UCred { /// /// This is only implemented under Linux, Android, iOS, macOS, Solaris and /// Illumos. On other platforms this will always return `None`. - pub fn pid(&self) -> Option<pid_t> { + pub fn pid(&self) -> Option<unix::pid_t> { self.pid } } @@ -48,7 +48,7 @@ pub(crate) use self::impl_solaris::get_peer_cred; #[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] pub(crate) mod impl_linux { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::{c_void, getsockopt, socklen_t, SOL_SOCKET, SO_PEERCRED}; use std::{io, mem}; @@ -87,9 +87,9 @@ pub(crate) mod impl_linux { ); if ret == 0 && ucred_size as usize == mem::size_of::<ucred>() { Ok(super::UCred { - uid: ucred.uid, - gid: ucred.gid, - pid: Some(ucred.pid), + uid: ucred.uid as unix::uid_t, + gid: ucred.gid as unix::gid_t, + pid: Some(ucred.pid as unix::pid_t), }) } else { Err(io::Error::last_os_error()) @@ -100,7 +100,7 @@ pub(crate) mod impl_linux { #[cfg(any(target_os = "netbsd"))] pub(crate) mod impl_netbsd { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::{c_void, getsockopt, socklen_t, unpcbid, LOCAL_PEEREID, SOL_SOCKET}; use std::io; @@ -129,9 +129,9 @@ pub(crate) mod impl_netbsd { ); if ret == 0 && unpcbid_size as usize == size_of::<unpcbid>() { Ok(super::UCred { - uid: unpcbid.unp_euid, - gid: unpcbid.unp_egid, - pid: Some(unpcbid.unp_pid), + uid: unpcbid.unp_euid as unix::uid_t, + gid: unpcbid.unp_egid as unix::gid_t, + pid: Some(unpcbid.unp_pid as unix::pid_t), }) } else { Err(io::Error::last_os_error()) @@ -142,7 +142,7 @@ pub(crate) mod impl_netbsd { #[cfg(any(target_os = "dragonfly", target_os = "freebsd"))] pub(crate) mod impl_bsd { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::getpeereid; use std::io; @@ -160,8 +160,8 @@ pub(crate) mod impl_bsd { if ret == 0 { Ok(super::UCred { - uid: uid.assume_init(), - gid: gid.assume_init(), + uid: uid.assume_init() as unix::uid_t, + gid: gid.assume_init() as unix::gid_t, pid: None, }) } else { @@ -173,7 +173,7 @@ pub(crate) mod impl_bsd { #[cfg(any(target_os = "macos", target_os = "ios"))] pub(crate) mod impl_macos { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::{c_void, getpeereid, getsockopt, pid_t, LOCAL_PEEREPID, SOL_LOCAL}; use std::io; @@ -207,9 +207,9 @@ pub(crate) mod impl_macos { if ret == 0 { Ok(super::UCred { - uid: uid.assume_init(), - gid: gid.assume_init(), - pid: Some(pid.assume_init()), + uid: uid.assume_init() as unix::uid_t, + gid: gid.assume_init() as unix::gid_t, + pid: Some(pid.assume_init() as unix::pid_t), }) } else { Err(io::Error::last_os_error()) @@ -220,7 +220,7 @@ pub(crate) mod impl_macos { #[cfg(any(target_os = "solaris", target_os = "illumos"))] pub(crate) mod impl_solaris { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use std::io; use std::os::unix::io::AsRawFd; use std::ptr; @@ -240,9 +240,9 @@ pub(crate) mod impl_solaris { libc::ucred_free(cred); Ok(super::UCred { - uid, - gid, - pid: Some(pid), + uid: uid as unix::uid_t, + gid: gid as unix::gid_t, + pid: Some(pid as unix::pid_t), }) } else { Err(io::Error::last_os_error()) diff --git a/src/net/windows/named_pipe.rs b/src/net/windows/named_pipe.rs index 550fd4d..9ede94e 100644 --- a/src/net/windows/named_pipe.rs +++ b/src/net/windows/named_pipe.rs @@ -12,25 +12,26 @@ use std::task::{Context, Poll}; use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready}; use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle}; +cfg_io_util! { + use bytes::BufMut; +} + // Hide imports which are not used when generating documentation. #[cfg(not(docsrs))] mod doc { pub(super) use crate::os::windows::ffi::OsStrExt; - pub(super) use crate::winapi::shared::minwindef::{DWORD, FALSE}; - pub(super) use crate::winapi::um::fileapi; - pub(super) use crate::winapi::um::handleapi; - pub(super) use crate::winapi::um::namedpipeapi; - pub(super) use crate::winapi::um::winbase; - pub(super) use crate::winapi::um::winnt; - + pub(super) mod windows_sys { + pub(crate) use windows_sys::{ + Win32::Foundation::*, Win32::Storage::FileSystem::*, Win32::System::Pipes::*, + Win32::System::SystemServices::*, + }; + } pub(super) use mio::windows as mio_windows; } // NB: none of these shows up in public API, so don't document them. #[cfg(docsrs)] mod doc { - pub type DWORD = crate::doc::NotDefinedHere; - pub(super) mod mio_windows { pub type NamedPipe = crate::doc::NotDefinedHere; } @@ -97,7 +98,6 @@ use self::doc::*; /// # Ok(()) } /// ``` /// -/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY /// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes #[derive(Debug)] pub struct NamedPipeServer { @@ -188,17 +188,15 @@ impl NamedPipeServer { /// # Ok(()) } /// ``` pub async fn connect(&self) -> io::Result<()> { - loop { - match self.io.connect() { - Ok(()) => break, - Err(e) if e.kind() == io::ErrorKind::WouldBlock => { - self.io.registration().readiness(Interest::WRITABLE).await?; - } - Err(e) => return Err(e), + match self.io.connect() { + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.io + .registration() + .async_io(Interest::WRITABLE, || self.io.connect()) + .await } + x => x, } - - Ok(()) } /// Disconnects the server end of a named pipe instance from a client @@ -207,7 +205,7 @@ impl NamedPipeServer { /// ``` /// use tokio::io::AsyncWriteExt; /// use tokio::net::windows::named_pipe::{ClientOptions, ServerOptions}; - /// use winapi::shared::winerror; + /// use windows_sys::Win32::Foundation::ERROR_PIPE_NOT_CONNECTED; /// /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-disconnect"; /// @@ -227,7 +225,7 @@ impl NamedPipeServer { /// // Write fails with an OS-specific error after client has been /// // disconnected. /// let e = client.write(b"ping").await.unwrap_err(); - /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_NOT_CONNECTED as i32)); + /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_NOT_CONNECTED as i32)); /// # Ok(()) } /// ``` pub fn disconnect(&self) -> io::Result<()> { @@ -240,6 +238,12 @@ impl NamedPipeServer { /// can be used to concurrently read / write to the same pipe on a single /// task without splitting the pipe. /// + /// The function may complete without the pipe being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Examples /// /// Concurrently read and write to the pipe on the same task without @@ -399,8 +403,12 @@ impl NamedPipeServer { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the pipe's read half is closed - /// and will no longer yield data. If the pipe is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The pipe's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the pipe is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -528,6 +536,86 @@ impl NamedPipeServer { .try_io(Interest::READABLE, || (&*self.io).read_vectored(bufs)) } + cfg_io_util! { + /// Tries to read data from the stream into the provided buffer, advancing the + /// buffer's internal cursor, returning how many bytes were read. + /// + /// Receives any pending data from the pipe but does not wait for new data + /// to arrive. On success, returns the number of bytes read. Because + /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by + /// the async task and can exist entirely on the stack. + /// + /// Usually, [`readable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: NamedPipeServer::readable() + /// [`ready()`]: NamedPipeServer::ready() + /// + /// # Return + /// + /// If data is successfully read, `Ok(n)` is returned, where `n` is the + /// number of bytes read. `Ok(0)` indicates the stream's read half is closed + /// and will no longer yield data. If the stream is not ready to read data + /// `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::windows::named_pipe; + /// use std::error::Error; + /// use std::io; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-client-readable"; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box<dyn Error>> { + /// let server = named_pipe::ServerOptions::new().create(PIPE_NAME)?; + /// + /// loop { + /// // Wait for the pipe to be readable + /// server.readable().await?; + /// + /// let mut buf = Vec::with_capacity(4096); + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match server.try_read_buf(&mut buf) { + /// Ok(0) => break, + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_read_buf<B: BufMut>(&self, buf: &mut B) -> io::Result<usize> { + self.io.registration().try_io(Interest::READABLE, || { + use std::io::Read; + + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]) }; + + // Safety: We trust `NamedPipeServer::read` to have filled up `n` bytes in the + // buffer. + let n = (&*self.io).read(dst)?; + + unsafe { + buf.advance_mut(n); + } + + Ok(n) + }) + } + } + /// Waits for the pipe to become writable. /// /// This function is equivalent to `ready(Interest::WRITABLE)` and is usually @@ -724,27 +812,32 @@ impl NamedPipeServer { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } - /// Tries to read or write from the socket using a user-provided IO operation. + /// Tries to read or write from the pipe using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// If the pipe is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the pipe by manually /// calling the appropriate syscall. If the operation fails because the - /// socket is not actually ready, then the closure should return a + /// pipe is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value /// of the closure is then returned by `try_io`. /// - /// If the socket is not ready, then the closure is not called + /// If the pipe is not ready, then the closure is not called /// and a `WouldBlock` error is returned. /// /// The closure should only return a `WouldBlock` error if it has performed - /// an IO operation on the socket that failed due to the socket not being + /// an IO operation on the pipe that failed due to the pipe not being /// ready. Returning a `WouldBlock` error in any other situation will - /// incorrectly clear the readiness flag, which can cause the socket to + /// incorrectly clear the readiness flag, which can cause the pipe to /// behave incorrectly. /// /// The closure should not perform the IO operation using any of the /// methods defined on the Tokio `NamedPipeServer` type, as this will mess with - /// the readiness flag and can cause the socket to behave incorrectly. + /// the readiness flag and can cause the pipe to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// @@ -819,7 +912,7 @@ impl AsRawHandle for NamedPipeServer { /// use std::time::Duration; /// use tokio::net::windows::named_pipe::ClientOptions; /// use tokio::time; -/// use winapi::shared::winerror; +/// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY; /// /// const PIPE_NAME: &str = r"\\.\pipe\named-pipe-idiomatic-client"; /// @@ -827,7 +920,7 @@ impl AsRawHandle for NamedPipeServer { /// let client = loop { /// match ClientOptions::new().open(PIPE_NAME) { /// Ok(client) => break client, -/// Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), +/// Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), /// Err(e) => return Err(e), /// } /// @@ -838,7 +931,7 @@ impl AsRawHandle for NamedPipeServer { /// # Ok(()) } /// ``` /// -/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY +/// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html /// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes #[derive(Debug)] pub struct NamedPipeClient { @@ -902,6 +995,12 @@ impl NamedPipeClient { /// can be used to concurrently read / write to the same pipe on a single /// task without splitting the pipe. /// + /// The function may complete without the pipe being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Examples /// /// Concurrently read and write to the pipe on the same task without @@ -1059,8 +1158,12 @@ impl NamedPipeClient { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the pipe's read half is closed - /// and will no longer yield data. If the pipe is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The pipe's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the pipe is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -1186,6 +1289,86 @@ impl NamedPipeClient { .try_io(Interest::READABLE, || (&*self.io).read_vectored(bufs)) } + cfg_io_util! { + /// Tries to read data from the stream into the provided buffer, advancing the + /// buffer's internal cursor, returning how many bytes were read. + /// + /// Receives any pending data from the pipe but does not wait for new data + /// to arrive. On success, returns the number of bytes read. Because + /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by + /// the async task and can exist entirely on the stack. + /// + /// Usually, [`readable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: NamedPipeClient::readable() + /// [`ready()`]: NamedPipeClient::ready() + /// + /// # Return + /// + /// If data is successfully read, `Ok(n)` is returned, where `n` is the + /// number of bytes read. `Ok(0)` indicates the stream's read half is closed + /// and will no longer yield data. If the stream is not ready to read data + /// `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::windows::named_pipe; + /// use std::error::Error; + /// use std::io; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-client-readable"; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box<dyn Error>> { + /// let client = named_pipe::ClientOptions::new().open(PIPE_NAME)?; + /// + /// loop { + /// // Wait for the pipe to be readable + /// client.readable().await?; + /// + /// let mut buf = Vec::with_capacity(4096); + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match client.try_read_buf(&mut buf) { + /// Ok(0) => break, + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_read_buf<B: BufMut>(&self, buf: &mut B) -> io::Result<usize> { + self.io.registration().try_io(Interest::READABLE, || { + use std::io::Read; + + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]) }; + + // Safety: We trust `NamedPipeClient::read` to have filled up `n` bytes in the + // buffer. + let n = (&*self.io).read(dst)?; + + unsafe { + buf.advance_mut(n); + } + + Ok(n) + }) + } + } + /// Waits for the pipe to become writable. /// /// This function is equivalent to `ready(Interest::WRITABLE)` and is usually @@ -1379,27 +1562,32 @@ impl NamedPipeClient { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } - /// Tries to read or write from the socket using a user-provided IO operation. + /// Tries to read or write from the pipe using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// If the pipe is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the pipe by manually /// calling the appropriate syscall. If the operation fails because the - /// socket is not actually ready, then the closure should return a + /// pipe is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value /// of the closure is then returned by `try_io`. /// - /// If the socket is not ready, then the closure is not called + /// If the pipe is not ready, then the closure is not called /// and a `WouldBlock` error is returned. /// /// The closure should only return a `WouldBlock` error if it has performed - /// an IO operation on the socket that failed due to the socket not being + /// an IO operation on the pipe that failed due to the pipe not being /// ready. Returning a `WouldBlock` error in any other situation will - /// incorrectly clear the readiness flag, which can cause the socket to + /// incorrectly clear the readiness flag, which can cause the pipe to /// behave incorrectly. /// /// The closure should not perform the IO operation using any of the methods /// defined on the Tokio `NamedPipeClient` type, as this will mess with the - /// readiness flag and can cause the socket to behave incorrectly. + /// readiness flag and can cause the pipe to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// @@ -1477,12 +1665,12 @@ macro_rules! bool_flag { /// See [`ServerOptions::create`]. #[derive(Debug, Clone)] pub struct ServerOptions { - open_mode: DWORD, - pipe_mode: DWORD, - max_instances: DWORD, - out_buffer_size: DWORD, - in_buffer_size: DWORD, - default_timeout: DWORD, + open_mode: u32, + pipe_mode: u32, + max_instances: u32, + out_buffer_size: u32, + in_buffer_size: u32, + default_timeout: u32, } impl ServerOptions { @@ -1499,9 +1687,9 @@ impl ServerOptions { /// ``` pub fn new() -> ServerOptions { ServerOptions { - open_mode: winbase::PIPE_ACCESS_DUPLEX | winbase::FILE_FLAG_OVERLAPPED, - pipe_mode: winbase::PIPE_TYPE_BYTE | winbase::PIPE_REJECT_REMOTE_CLIENTS, - max_instances: winbase::PIPE_UNLIMITED_INSTANCES, + open_mode: windows_sys::PIPE_ACCESS_DUPLEX | windows_sys::FILE_FLAG_OVERLAPPED, + pipe_mode: windows_sys::PIPE_TYPE_BYTE | windows_sys::PIPE_REJECT_REMOTE_CLIENTS, + max_instances: windows_sys::PIPE_UNLIMITED_INSTANCES, out_buffer_size: 65536, in_buffer_size: 65536, default_timeout: 0, @@ -1517,11 +1705,10 @@ impl ServerOptions { /// /// [`dwPipeMode`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea pub fn pipe_mode(&mut self, pipe_mode: PipeMode) -> &mut Self { - self.pipe_mode = match pipe_mode { - PipeMode::Byte => winbase::PIPE_TYPE_BYTE, - PipeMode::Message => winbase::PIPE_TYPE_MESSAGE, - }; - + let is_msg = matches!(pipe_mode, PipeMode::Message); + // Pipe mode is implemented as a bit flag 0x4. Set is message and unset + // is byte. + bool_flag!(self.pipe_mode, is_msg, windows_sys::PIPE_TYPE_MESSAGE); self } @@ -1617,7 +1804,7 @@ impl ServerOptions { /// # Ok(()) } /// ``` pub fn access_inbound(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_INBOUND); + bool_flag!(self.open_mode, allowed, windows_sys::PIPE_ACCESS_INBOUND); self } @@ -1715,7 +1902,7 @@ impl ServerOptions { /// # Ok(()) } /// ``` pub fn access_outbound(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_OUTBOUND); + bool_flag!(self.open_mode, allowed, windows_sys::PIPE_ACCESS_OUTBOUND); self } @@ -1786,7 +1973,113 @@ impl ServerOptions { bool_flag!( self.open_mode, first, - winbase::FILE_FLAG_FIRST_PIPE_INSTANCE + windows_sys::FILE_FLAG_FIRST_PIPE_INSTANCE + ); + self + } + + /// Requests permission to modify the pipe's discretionary access control list. + /// + /// This corresponds to setting [`WRITE_DAC`] in dwOpenMode. + /// + /// # Examples + /// + /// ``` + /// use std::{io, os::windows::prelude::AsRawHandle, ptr}; + // + /// use tokio::net::windows::named_pipe::ServerOptions; + /// use windows_sys::{ + /// Win32::Foundation::ERROR_SUCCESS, + /// Win32::Security::DACL_SECURITY_INFORMATION, + /// Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT}, + /// }; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe"; + /// + /// # #[tokio::main] async fn main() -> io::Result<()> { + /// let mut pipe_template = ServerOptions::new(); + /// pipe_template.write_dac(true); + /// let pipe = pipe_template.create(PIPE_NAME)?; + /// + /// unsafe { + /// assert_eq!( + /// ERROR_SUCCESS, + /// SetSecurityInfo( + /// pipe.as_raw_handle() as _, + /// SE_KERNEL_OBJECT, + /// DACL_SECURITY_INFORMATION, + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ) + /// ); + /// } + /// + /// # Ok(()) } + /// ``` + /// + /// ``` + /// use std::{io, os::windows::prelude::AsRawHandle, ptr}; + // + /// use tokio::net::windows::named_pipe::ServerOptions; + /// use windows_sys::{ + /// Win32::Foundation::ERROR_ACCESS_DENIED, + /// Win32::Security::DACL_SECURITY_INFORMATION, + /// Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT}, + /// }; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe_fail"; + /// + /// # #[tokio::main] async fn main() -> io::Result<()> { + /// let mut pipe_template = ServerOptions::new(); + /// pipe_template.write_dac(false); + /// let pipe = pipe_template.create(PIPE_NAME)?; + /// + /// unsafe { + /// assert_eq!( + /// ERROR_ACCESS_DENIED, + /// SetSecurityInfo( + /// pipe.as_raw_handle() as _, + /// SE_KERNEL_OBJECT, + /// DACL_SECURITY_INFORMATION, + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ) + /// ); + /// } + /// + /// # Ok(()) } + /// ``` + /// + /// [`WRITE_DAC`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea + pub fn write_dac(&mut self, requested: bool) -> &mut Self { + bool_flag!(self.open_mode, requested, windows_sys::WRITE_DAC); + self + } + + /// Requests permission to modify the pipe's owner. + /// + /// This corresponds to setting [`WRITE_OWNER`] in dwOpenMode. + /// + /// [`WRITE_OWNER`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea + pub fn write_owner(&mut self, requested: bool) -> &mut Self { + bool_flag!(self.open_mode, requested, windows_sys::WRITE_OWNER); + self + } + + /// Requests permission to modify the pipe's system access control list. + /// + /// This corresponds to setting [`ACCESS_SYSTEM_SECURITY`] in dwOpenMode. + /// + /// [`ACCESS_SYSTEM_SECURITY`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea + pub fn access_system_security(&mut self, requested: bool) -> &mut Self { + bool_flag!( + self.open_mode, + requested, + windows_sys::ACCESS_SYSTEM_SECURITY ); self } @@ -1798,7 +2091,11 @@ impl ServerOptions { /// /// [`PIPE_REJECT_REMOTE_CLIENTS`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea#pipe_reject_remote_clients pub fn reject_remote_clients(&mut self, reject: bool) -> &mut Self { - bool_flag!(self.pipe_mode, reject, winbase::PIPE_REJECT_REMOTE_CLIENTS); + bool_flag!( + self.pipe_mode, + reject, + windows_sys::PIPE_REJECT_REMOTE_CLIENTS + ); self } @@ -1820,7 +2117,7 @@ impl ServerOptions { /// ``` /// use std::io; /// use tokio::net::windows::named_pipe::{ServerOptions, ClientOptions}; - /// use winapi::shared::winerror; + /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY; /// /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-max-instances"; /// @@ -1836,11 +2133,11 @@ impl ServerOptions { /// /// // Too many servers! /// let e = server.create(PIPE_NAME).unwrap_err(); - /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32)); + /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32)); /// /// // Still too many servers even if we specify a higher value! /// let e = server.max_instances(100).create(PIPE_NAME).unwrap_err(); - /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32)); + /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32)); /// # Ok(()) } /// ``` /// @@ -1856,9 +2153,10 @@ impl ServerOptions { /// let builder = ServerOptions::new().max_instances(255); /// # Ok(()) } /// ``` + #[track_caller] pub fn max_instances(&mut self, instances: usize) -> &mut Self { assert!(instances < 255, "cannot specify more than 254 instances"); - self.max_instances = instances as DWORD; + self.max_instances = instances as u32; self } @@ -1868,7 +2166,7 @@ impl ServerOptions { /// /// [`nOutBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea pub fn out_buffer_size(&mut self, buffer: u32) -> &mut Self { - self.out_buffer_size = buffer as DWORD; + self.out_buffer_size = buffer; self } @@ -1878,7 +2176,7 @@ impl ServerOptions { /// /// [`nInBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea pub fn in_buffer_size(&mut self, buffer: u32) -> &mut Self { - self.in_buffer_size = buffer as DWORD; + self.in_buffer_size = buffer; self } @@ -1935,7 +2233,7 @@ impl ServerOptions { /// /// [`create`]: ServerOptions::create /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew - /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES + /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html pub unsafe fn create_with_security_attributes_raw( &self, addr: impl AsRef<OsStr>, @@ -1943,7 +2241,7 @@ impl ServerOptions { ) -> io::Result<NamedPipeServer> { let addr = encode_addr(addr); - let h = namedpipeapi::CreateNamedPipeW( + let h = windows_sys::CreateNamedPipeW( addr.as_ptr(), self.open_mode, self.pipe_mode, @@ -1954,11 +2252,11 @@ impl ServerOptions { attrs as *mut _, ); - if h == handleapi::INVALID_HANDLE_VALUE { + if h == windows_sys::INVALID_HANDLE_VALUE { return Err(io::Error::last_os_error()); } - NamedPipeServer::from_raw_handle(h) + NamedPipeServer::from_raw_handle(h as _) } } @@ -1968,8 +2266,8 @@ impl ServerOptions { /// See [`ClientOptions::open`]. #[derive(Debug, Clone)] pub struct ClientOptions { - desired_access: DWORD, - security_qos_flags: DWORD, + desired_access: u32, + security_qos_flags: u32, } impl ClientOptions { @@ -1988,8 +2286,9 @@ impl ClientOptions { /// ``` pub fn new() -> Self { Self { - desired_access: winnt::GENERIC_READ | winnt::GENERIC_WRITE, - security_qos_flags: winbase::SECURITY_IDENTIFICATION | winbase::SECURITY_SQOS_PRESENT, + desired_access: windows_sys::GENERIC_READ | windows_sys::GENERIC_WRITE, + security_qos_flags: windows_sys::SECURITY_IDENTIFICATION + | windows_sys::SECURITY_SQOS_PRESENT, } } @@ -2000,7 +2299,7 @@ impl ClientOptions { /// [`GENERIC_READ`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew pub fn read(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.desired_access, allowed, winnt::GENERIC_READ); + bool_flag!(self.desired_access, allowed, windows_sys::GENERIC_READ); self } @@ -2011,7 +2310,7 @@ impl ClientOptions { /// [`GENERIC_WRITE`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew pub fn write(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.desired_access, allowed, winnt::GENERIC_WRITE); + bool_flag!(self.desired_access, allowed, windows_sys::GENERIC_WRITE); self } @@ -2034,11 +2333,11 @@ impl ClientOptions { /// automatically when using this method. /// /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea - /// [`SECURITY_IDENTIFICATION`]: crate::winapi::um::winbase::SECURITY_IDENTIFICATION + /// [`SECURITY_IDENTIFICATION`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Storage/FileSystem/constant.SECURITY_IDENTIFICATION.html /// [Impersonation Levels]: https://docs.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level pub fn security_qos_flags(&mut self, flags: u32) -> &mut Self { // See: https://github.com/rust-lang/rust/pull/58216 - self.security_qos_flags = flags | winbase::SECURITY_SQOS_PRESENT; + self.security_qos_flags = flags | windows_sys::SECURITY_SQOS_PRESENT; self } @@ -2063,8 +2362,7 @@ impl ClientOptions { /// but the server is not currently waiting for a connection. Please see the /// examples for how to check for this error. /// - /// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY - /// [`winapi`]: crate::winapi + /// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html /// [enabled I/O]: crate::runtime::Builder::enable_io /// [Tokio Runtime]: crate::runtime::Runtime /// @@ -2075,7 +2373,7 @@ impl ClientOptions { /// use std::time::Duration; /// use tokio::net::windows::named_pipe::ClientOptions; /// use tokio::time; - /// use winapi::shared::winerror; + /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY; /// /// const PIPE_NAME: &str = r"\\.\pipe\mynamedpipe"; /// @@ -2083,7 +2381,7 @@ impl ClientOptions { /// let client = loop { /// match ClientOptions::new().open(PIPE_NAME) { /// Ok(client) => break client, - /// Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), + /// Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), /// Err(e) => return Err(e), /// } /// @@ -2113,7 +2411,7 @@ impl ClientOptions { /// /// [`open`]: ClientOptions::open /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew - /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES + /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html pub unsafe fn open_with_security_attributes_raw( &self, addr: impl AsRef<OsStr>, @@ -2122,28 +2420,28 @@ impl ClientOptions { let addr = encode_addr(addr); // NB: We could use a platform specialized `OpenOptions` here, but since - // we have access to winapi it ultimately doesn't hurt to use + // we have access to windows_sys it ultimately doesn't hurt to use // `CreateFile` explicitly since it allows the use of our already // well-structured wide `addr` to pass into CreateFileW. - let h = fileapi::CreateFileW( + let h = windows_sys::CreateFileW( addr.as_ptr(), self.desired_access, 0, attrs as *mut _, - fileapi::OPEN_EXISTING, + windows_sys::OPEN_EXISTING, self.get_flags(), - ptr::null_mut(), + 0, ); - if h == handleapi::INVALID_HANDLE_VALUE { + if h == windows_sys::INVALID_HANDLE_VALUE { return Err(io::Error::last_os_error()); } - NamedPipeClient::from_raw_handle(h) + NamedPipeClient::from_raw_handle(h as _) } fn get_flags(&self) -> u32 { - self.security_qos_flags | winbase::FILE_FLAG_OVERLAPPED + self.security_qos_flags | windows_sys::FILE_FLAG_OVERLAPPED } } @@ -2156,16 +2454,19 @@ pub enum PipeMode { /// Data is written to the pipe as a stream of bytes. The pipe does not /// distinguish bytes written during different write operations. /// - /// Corresponds to [`PIPE_TYPE_BYTE`][crate::winapi::um::winbase::PIPE_TYPE_BYTE]. + /// Corresponds to [`PIPE_TYPE_BYTE`]. + /// + /// [`PIPE_TYPE_BYTE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_BYTE.html Byte, /// Data is written to the pipe as a stream of messages. The pipe treats the /// bytes written during each write operation as a message unit. Any reading /// on a named pipe returns [`ERROR_MORE_DATA`] when a message is not read /// completely. /// - /// Corresponds to [`PIPE_TYPE_MESSAGE`][crate::winapi::um::winbase::PIPE_TYPE_MESSAGE]. + /// Corresponds to [`PIPE_TYPE_MESSAGE`]. /// - /// [`ERROR_MORE_DATA`]: crate::winapi::shared::winerror::ERROR_MORE_DATA + /// [`ERROR_MORE_DATA`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_MORE_DATA.html + /// [`PIPE_TYPE_MESSAGE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_MESSAGE.html Message, } @@ -2175,11 +2476,15 @@ pub enum PipeMode { pub enum PipeEnd { /// The named pipe refers to the client end of a named pipe instance. /// - /// Corresponds to [`PIPE_CLIENT_END`][crate::winapi::um::winbase::PIPE_CLIENT_END]. + /// Corresponds to [`PIPE_CLIENT_END`]. + /// + /// [`PIPE_CLIENT_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_CLIENT_END.html Client, /// The named pipe refers to the server end of a named pipe instance. /// - /// Corresponds to [`PIPE_SERVER_END`][crate::winapi::um::winbase::PIPE_SERVER_END]. + /// Corresponds to [`PIPE_SERVER_END`]. + /// + /// [`PIPE_SERVER_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_SERVER_END.html Server, } @@ -2217,26 +2522,26 @@ unsafe fn named_pipe_info(handle: RawHandle) -> io::Result<PipeInfo> { let mut in_buffer_size = 0; let mut max_instances = 0; - let result = namedpipeapi::GetNamedPipeInfo( - handle, + let result = windows_sys::GetNamedPipeInfo( + handle as _, &mut flags, &mut out_buffer_size, &mut in_buffer_size, &mut max_instances, ); - if result == FALSE { + if result == 0 { return Err(io::Error::last_os_error()); } let mut end = PipeEnd::Client; let mut mode = PipeMode::Byte; - if flags & winbase::PIPE_SERVER_END != 0 { + if flags & windows_sys::PIPE_SERVER_END != 0 { end = PipeEnd::Server; } - if flags & winbase::PIPE_TYPE_MESSAGE != 0 { + if flags & windows_sys::PIPE_TYPE_MESSAGE != 0 { mode = PipeMode::Message; } @@ -2248,3 +2553,48 @@ unsafe fn named_pipe_info(handle: RawHandle) -> io::Result<PipeInfo> { max_instances, }) } + +#[cfg(test)] +mod test { + use self::windows_sys::{PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_TYPE_MESSAGE}; + use super::*; + + #[test] + fn opts_default_pipe_mode() { + let opts = ServerOptions::new(); + assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS); + } + + #[test] + fn opts_unset_reject_remote() { + let mut opts = ServerOptions::new(); + opts.reject_remote_clients(false); + assert_eq!(opts.pipe_mode & PIPE_REJECT_REMOTE_CLIENTS, 0); + } + + #[test] + fn opts_set_pipe_mode_maintains_reject_remote_clients() { + let mut opts = ServerOptions::new(); + opts.pipe_mode(PipeMode::Byte); + assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS); + + opts.reject_remote_clients(false); + opts.pipe_mode(PipeMode::Byte); + assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE); + + opts.reject_remote_clients(true); + opts.pipe_mode(PipeMode::Byte); + assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS); + + opts.reject_remote_clients(false); + opts.pipe_mode(PipeMode::Message); + assert_eq!(opts.pipe_mode, PIPE_TYPE_MESSAGE); + + opts.reject_remote_clients(true); + opts.pipe_mode(PipeMode::Message); + assert_eq!( + opts.pipe_mode, + PIPE_TYPE_MESSAGE | PIPE_REJECT_REMOTE_CLIENTS + ); + } +} diff --git a/src/park/either.rs b/src/park/either.rs deleted file mode 100644 index ee02ec1..0000000 --- a/src/park/either.rs +++ /dev/null @@ -1,74 +0,0 @@ -#![cfg_attr(not(feature = "full"), allow(dead_code))] - -use crate::park::{Park, Unpark}; - -use std::fmt; -use std::time::Duration; - -pub(crate) enum Either<A, B> { - A(A), - B(B), -} - -impl<A, B> Park for Either<A, B> -where - A: Park, - B: Park, -{ - type Unpark = Either<A::Unpark, B::Unpark>; - type Error = Either<A::Error, B::Error>; - - fn unpark(&self) -> Self::Unpark { - match self { - Either::A(a) => Either::A(a.unpark()), - Either::B(b) => Either::B(b.unpark()), - } - } - - fn park(&mut self) -> Result<(), Self::Error> { - match self { - Either::A(a) => a.park().map_err(Either::A), - Either::B(b) => b.park().map_err(Either::B), - } - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - match self { - Either::A(a) => a.park_timeout(duration).map_err(Either::A), - Either::B(b) => b.park_timeout(duration).map_err(Either::B), - } - } - - fn shutdown(&mut self) { - match self { - Either::A(a) => a.shutdown(), - Either::B(b) => b.shutdown(), - } - } -} - -impl<A, B> Unpark for Either<A, B> -where - A: Unpark, - B: Unpark, -{ - fn unpark(&self) { - match self { - Either::A(a) => a.unpark(), - Either::B(b) => b.unpark(), - } - } -} - -impl<A, B> fmt::Debug for Either<A, B> -where - A: fmt::Debug, - B: fmt::Debug, -{ - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Either::A(a) => a.fmt(fmt), - Either::B(b) => b.fmt(fmt), - } - } -} diff --git a/src/park/mod.rs b/src/park/mod.rs deleted file mode 100644 index 87d04ff..0000000 --- a/src/park/mod.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Abstraction over blocking and unblocking the current thread. -//! -//! Provides an abstraction over blocking the current thread. This is similar to -//! the park / unpark constructs provided by `std` but made generic. This allows -//! embedding custom functionality to perform when the thread is blocked. -//! -//! A blocked `Park` instance is unblocked by calling `unpark` on its -//! `Unpark` handle. -//! -//! The `ParkThread` struct implements `Park` using `thread::park` to put the -//! thread to sleep. The Tokio reactor also implements park, but uses -//! `mio::Poll` to block the thread instead. -//! -//! The `Park` trait is composable. A timer implementation might decorate a -//! `Park` implementation by checking if any timeouts have elapsed after the -//! inner `Park` implementation unblocks. -//! -//! # Model -//! -//! Conceptually, each `Park` instance has an associated token, which is -//! initially not present: -//! -//! * The `park` method blocks the current thread unless or until the token is -//! available, at which point it atomically consumes the token. -//! * The `unpark` method atomically makes the token available if it wasn't -//! already. -//! -//! Some things to note: -//! -//! * If `unpark` is called before `park`, the next call to `park` will -//! **not** block the thread. -//! * **Spurious** wakeups are permitted, i.e., the `park` method may unblock -//! even if `unpark` was not called. -//! * `park_timeout` does the same as `park` but allows specifying a maximum -//! time to block the thread for. - -cfg_rt! { - pub(crate) mod either; -} - -#[cfg(any(feature = "rt", feature = "sync"))] -pub(crate) mod thread; - -use std::fmt::Debug; -use std::sync::Arc; -use std::time::Duration; - -/// Blocks the current thread. -pub(crate) trait Park { - /// Unpark handle type for the `Park` implementation. - type Unpark: Unpark; - - /// Error returned by `park`. - type Error: Debug; - - /// Gets a new `Unpark` handle associated with this `Park` instance. - fn unpark(&self) -> Self::Unpark; - - /// Blocks the current thread unless or until the token is available. - /// - /// A call to `park` does not guarantee that the thread will remain blocked - /// forever, and callers should be prepared for this possibility. This - /// function may wakeup spuriously for any reason. - /// - /// # Panics - /// - /// This function **should** not panic, but ultimately, panics are left as - /// an implementation detail. Refer to the documentation for the specific - /// `Park` implementation. - fn park(&mut self) -> Result<(), Self::Error>; - - /// Parks the current thread for at most `duration`. - /// - /// This function is the same as `park` but allows specifying a maximum time - /// to block the thread for. - /// - /// Same as `park`, there is no guarantee that the thread will remain - /// blocked for any amount of time. Spurious wakeups are permitted for any - /// reason. - /// - /// # Panics - /// - /// This function **should** not panic, but ultimately, panics are left as - /// an implementation detail. Refer to the documentation for the specific - /// `Park` implementation. - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error>; - - /// Releases all resources holded by the parker for proper leak-free shutdown. - fn shutdown(&mut self); -} - -/// Unblock a thread blocked by the associated `Park` instance. -pub(crate) trait Unpark: Sync + Send + 'static { - /// Unblocks a thread that is blocked by the associated `Park` handle. - /// - /// Calling `unpark` atomically makes available the unpark token, if it is - /// not already available. - /// - /// # Panics - /// - /// This function **should** not panic, but ultimately, panics are left as - /// an implementation detail. Refer to the documentation for the specific - /// `Unpark` implementation. - fn unpark(&self); -} - -impl Unpark for Box<dyn Unpark> { - fn unpark(&self) { - (**self).unpark() - } -} - -impl Unpark for Arc<dyn Unpark> { - fn unpark(&self) { - (**self).unpark() - } -} diff --git a/src/park/thread.rs b/src/park/thread.rs deleted file mode 100644 index 27ce202..0000000 --- a/src/park/thread.rs +++ /dev/null @@ -1,346 +0,0 @@ -#![cfg_attr(not(feature = "full"), allow(dead_code))] - -use crate::loom::sync::atomic::AtomicUsize; -use crate::loom::sync::{Arc, Condvar, Mutex}; -use crate::park::{Park, Unpark}; - -use std::sync::atomic::Ordering::SeqCst; -use std::time::Duration; - -#[derive(Debug)] -pub(crate) struct ParkThread { - inner: Arc<Inner>, -} - -pub(crate) type ParkError = (); - -/// Unblocks a thread that was blocked by `ParkThread`. -#[derive(Clone, Debug)] -pub(crate) struct UnparkThread { - inner: Arc<Inner>, -} - -#[derive(Debug)] -struct Inner { - state: AtomicUsize, - mutex: Mutex<()>, - condvar: Condvar, -} - -const EMPTY: usize = 0; -const PARKED: usize = 1; -const NOTIFIED: usize = 2; - -thread_local! { - static CURRENT_PARKER: ParkThread = ParkThread::new(); -} - -// ==== impl ParkThread ==== - -impl ParkThread { - pub(crate) fn new() -> Self { - Self { - inner: Arc::new(Inner { - state: AtomicUsize::new(EMPTY), - mutex: Mutex::new(()), - condvar: Condvar::new(), - }), - } - } -} - -impl Park for ParkThread { - type Unpark = UnparkThread; - type Error = ParkError; - - fn unpark(&self) -> Self::Unpark { - let inner = self.inner.clone(); - UnparkThread { inner } - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.inner.park(); - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.inner.park_timeout(duration); - Ok(()) - } - - fn shutdown(&mut self) { - self.inner.shutdown(); - } -} - -// ==== impl Inner ==== - -impl Inner { - /// Parks the current thread for at most `dur`. - fn park(&self) { - // If we were previously notified then we consume this notification and - // return quickly. - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - return; - } - - // Otherwise we need to coordinate going to sleep - let mut m = self.mutex.lock(); - - match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { - Ok(_) => {} - Err(NOTIFIED) => { - // We must read here, even though we know it will be `NOTIFIED`. - // This is because `unpark` may have been called again since we read - // `NOTIFIED` in the `compare_exchange` above. We must perform an - // acquire operation that synchronizes with that `unpark` to observe - // any writes it made before the call to unpark. To do that we must - // read from the write it made to `state`. - let old = self.state.swap(EMPTY, SeqCst); - debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); - - return; - } - Err(actual) => panic!("inconsistent park state; actual = {}", actual), - } - - loop { - m = self.condvar.wait(m).unwrap(); - - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - // got a notification - return; - } - - // spurious wakeup, go back to sleep - } - } - - fn park_timeout(&self, dur: Duration) { - // Like `park` above we have a fast path for an already-notified thread, - // and afterwards we start coordinating for a sleep. Return quickly. - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - return; - } - - if dur == Duration::from_millis(0) { - return; - } - - let m = self.mutex.lock(); - - match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { - Ok(_) => {} - Err(NOTIFIED) => { - // We must read again here, see `park`. - let old = self.state.swap(EMPTY, SeqCst); - debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); - - return; - } - Err(actual) => panic!("inconsistent park_timeout state; actual = {}", actual), - } - - // Wait with a timeout, and if we spuriously wake up or otherwise wake up - // from a notification, we just want to unconditionally set the state back to - // empty, either consuming a notification or un-flagging ourselves as - // parked. - let (_m, _result) = self.condvar.wait_timeout(m, dur).unwrap(); - - match self.state.swap(EMPTY, SeqCst) { - NOTIFIED => {} // got a notification, hurray! - PARKED => {} // no notification, alas - n => panic!("inconsistent park_timeout state: {}", n), - } - } - - fn unpark(&self) { - // To ensure the unparked thread will observe any writes we made before - // this call, we must perform a release operation that `park` can - // synchronize with. To do that we must write `NOTIFIED` even if `state` - // is already `NOTIFIED`. That is why this must be a swap rather than a - // compare-and-swap that returns if it reads `NOTIFIED` on failure. - match self.state.swap(NOTIFIED, SeqCst) { - EMPTY => return, // no one was waiting - NOTIFIED => return, // already unparked - PARKED => {} // gotta go wake someone up - _ => panic!("inconsistent state in unpark"), - } - - // There is a period between when the parked thread sets `state` to - // `PARKED` (or last checked `state` in the case of a spurious wake - // up) and when it actually waits on `cvar`. If we were to notify - // during this period it would be ignored and then when the parked - // thread went to sleep it would never wake up. Fortunately, it has - // `lock` locked at this stage so we can acquire `lock` to wait until - // it is ready to receive the notification. - // - // Releasing `lock` before the call to `notify_one` means that when the - // parked thread wakes it doesn't get woken only to have to wait for us - // to release `lock`. - drop(self.mutex.lock()); - - self.condvar.notify_one() - } - - fn shutdown(&self) { - self.condvar.notify_all(); - } -} - -impl Default for ParkThread { - fn default() -> Self { - Self::new() - } -} - -// ===== impl UnparkThread ===== - -impl Unpark for UnparkThread { - fn unpark(&self) { - self.inner.unpark(); - } -} - -use std::future::Future; -use std::marker::PhantomData; -use std::mem; -use std::rc::Rc; -use std::task::{RawWaker, RawWakerVTable, Waker}; - -/// Blocks the current thread using a condition variable. -#[derive(Debug)] -pub(crate) struct CachedParkThread { - _anchor: PhantomData<Rc<()>>, -} - -impl CachedParkThread { - /// Creates a new `ParkThread` handle for the current thread. - /// - /// This type cannot be moved to other threads, so it should be created on - /// the thread that the caller intends to park. - pub(crate) fn new() -> CachedParkThread { - CachedParkThread { - _anchor: PhantomData, - } - } - - pub(crate) fn get_unpark(&self) -> Result<UnparkThread, ParkError> { - self.with_current(|park_thread| park_thread.unpark()) - } - - /// Gets a reference to the `ParkThread` handle for this thread. - fn with_current<F, R>(&self, f: F) -> Result<R, ParkError> - where - F: FnOnce(&ParkThread) -> R, - { - CURRENT_PARKER.try_with(|inner| f(inner)).map_err(|_| ()) - } - - pub(crate) fn block_on<F: Future>(&mut self, f: F) -> Result<F::Output, ParkError> { - use std::task::Context; - use std::task::Poll::Ready; - - // `get_unpark()` should not return a Result - let waker = self.get_unpark()?.into_waker(); - let mut cx = Context::from_waker(&waker); - - pin!(f); - - loop { - if let Ready(v) = crate::coop::budget(|| f.as_mut().poll(&mut cx)) { - return Ok(v); - } - - self.park()?; - } - } -} - -impl Park for CachedParkThread { - type Unpark = UnparkThread; - type Error = ParkError; - - fn unpark(&self) -> Self::Unpark { - self.get_unpark().unwrap() - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.with_current(|park_thread| park_thread.inner.park())?; - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.with_current(|park_thread| park_thread.inner.park_timeout(duration))?; - Ok(()) - } - - fn shutdown(&mut self) { - let _ = self.with_current(|park_thread| park_thread.inner.shutdown()); - } -} - -impl UnparkThread { - pub(crate) fn into_waker(self) -> Waker { - unsafe { - let raw = unparker_to_raw_waker(self.inner); - Waker::from_raw(raw) - } - } -} - -impl Inner { - #[allow(clippy::wrong_self_convention)] - fn into_raw(this: Arc<Inner>) -> *const () { - Arc::into_raw(this) as *const () - } - - unsafe fn from_raw(ptr: *const ()) -> Arc<Inner> { - Arc::from_raw(ptr as *const Inner) - } -} - -unsafe fn unparker_to_raw_waker(unparker: Arc<Inner>) -> RawWaker { - RawWaker::new( - Inner::into_raw(unparker), - &RawWakerVTable::new(clone, wake, wake_by_ref, drop_waker), - ) -} - -unsafe fn clone(raw: *const ()) -> RawWaker { - let unparker = Inner::from_raw(raw); - - // Increment the ref count - mem::forget(unparker.clone()); - - unparker_to_raw_waker(unparker) -} - -unsafe fn drop_waker(raw: *const ()) { - let _ = Inner::from_raw(raw); -} - -unsafe fn wake(raw: *const ()) { - let unparker = Inner::from_raw(raw); - unparker.unpark(); -} - -unsafe fn wake_by_ref(raw: *const ()) { - let unparker = Inner::from_raw(raw); - unparker.unpark(); - - // We don't actually own a reference to the unparker - mem::forget(unparker); -} diff --git a/src/process/mod.rs b/src/process/mod.rs index 6eeefdb..66e4212 100644 --- a/src/process/mod.rs +++ b/src/process/mod.rs @@ -97,6 +97,59 @@ //! } //! ``` //! +//! Here is another example using `sort` writing into the child process +//! standard input, capturing the output of the sorted text. +//! +//! ```no_run +//! use tokio::io::AsyncWriteExt; +//! use tokio::process::Command; +//! +//! use std::process::Stdio; +//! +//! #[tokio::main] +//! async fn main() -> Result<(), Box<dyn std::error::Error>> { +//! let mut cmd = Command::new("sort"); +//! +//! // Specifying that we want pipe both the output and the input. +//! // Similarly to capturing the output, by configuring the pipe +//! // to stdin it can now be used as an asynchronous writer. +//! cmd.stdout(Stdio::piped()); +//! cmd.stdin(Stdio::piped()); +//! +//! let mut child = cmd.spawn().expect("failed to spawn command"); +//! +//! // These are the animals we want to sort +//! let animals: &[&str] = &["dog", "bird", "frog", "cat", "fish"]; +//! +//! let mut stdin = child +//! .stdin +//! .take() +//! .expect("child did not have a handle to stdin"); +//! +//! // Write our animals to the child process +//! // Note that the behavior of `sort` is to buffer _all input_ before writing any output. +//! // In the general sense, it is recommended to write to the child in a separate task as +//! // awaiting its exit (or output) to avoid deadlocks (for example, the child tries to write +//! // some output but gets stuck waiting on the parent to read from it, meanwhile the parent +//! // is stuck waiting to write its input completely before reading the output). +//! stdin +//! .write(animals.join("\n").as_bytes()) +//! .await +//! .expect("could not write to stdin"); +//! +//! // We drop the handle here which signals EOF to the child process. +//! // This tells the child process that it there is no more data on the pipe. +//! drop(stdin); +//! +//! let op = child.wait_with_output().await?; +//! +//! // Results should come back in sorted order +//! assert_eq!(op.stdout, "bird\ncat\ndog\nfish\nfrog\n".as_bytes()); +//! +//! Ok(()) +//! } +//! ``` +//! //! With some coordination, we can also pipe the output of one command into //! another. //! @@ -264,6 +317,12 @@ impl Command { Self::from(StdCommand::new(program)) } + /// Cheaply convert to a `&std::process::Command` for places where the type from the standard + /// library is expected. + pub fn as_std(&self) -> &StdCommand { + &self.std + } + /// Adds an argument to pass to the program. /// /// Only one argument can be passed per use. So instead of: @@ -631,6 +690,36 @@ impl Command { self } + /// Sets the process group ID (PGID) of the child process. Equivalent to a + /// setpgid call in the child process, but may be more efficient. + /// + /// Process groups determine which processes receive signals. + /// + /// **Note**: This is an [unstable API][unstable] but will be stabilised once + /// tokio's MSRV is sufficiently new. See [the documentation on + /// unstable features][unstable] for details about using unstable features. + /// + /// If you want similar behaviour without using this unstable feature you can + /// create a [`std::process::Command`] and convert that into a + /// [`tokio::process::Command`] using the `From` trait. + /// + /// [unstable]: crate#unstable-features + /// [`tokio::process::Command`]: crate::process::Command + /// + /// ```no_run + /// use tokio::process::Command; + /// + /// let command = Command::new("ls") + /// .process_group(0); + /// ``` + #[cfg(unix)] + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(all(unix, tokio_unstable))))] + pub fn process_group(&mut self, pgroup: i32) -> &mut Command { + self.std.process_group(pgroup); + self + } + /// Executes the command as a child process, returning a handle to it. /// /// By default, stdin, stdout and stderr are inherited from the parent. @@ -865,7 +954,7 @@ where fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let ret = Pin::new(&mut self.inner).poll(cx); @@ -1124,19 +1213,26 @@ impl Child { pub async fn wait_with_output(mut self) -> io::Result<Output> { use crate::future::try_join3; - async fn read_to_end<A: AsyncRead + Unpin>(io: Option<A>) -> io::Result<Vec<u8>> { + async fn read_to_end<A: AsyncRead + Unpin>(io: &mut Option<A>) -> io::Result<Vec<u8>> { let mut vec = Vec::new(); - if let Some(mut io) = io { - crate::io::util::read_to_end(&mut io, &mut vec).await?; + if let Some(io) = io.as_mut() { + crate::io::util::read_to_end(io, &mut vec).await?; } Ok(vec) } - let stdout_fut = read_to_end(self.stdout.take()); - let stderr_fut = read_to_end(self.stderr.take()); + let mut stdout_pipe = self.stdout.take(); + let mut stderr_pipe = self.stderr.take(); + + let stdout_fut = read_to_end(&mut stdout_pipe); + let stderr_fut = read_to_end(&mut stderr_pipe); let (status, stdout, stderr) = try_join3(self.wait(), stdout_fut, stderr_fut).await?; + // Drop happens after `try_join` due to <https://github.com/tokio-rs/tokio/issues/4309> + drop(stdout_pipe); + drop(stderr_pipe); + Ok(Output { status, stdout, @@ -1219,41 +1315,51 @@ impl ChildStderr { impl AsyncWrite for ChildStdin { fn poll_write( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8], ) -> Poll<io::Result<usize>> { - self.inner.poll_write(cx, buf) + Pin::new(&mut self.inner).poll_write(cx, buf) } - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> { - Poll::Ready(Ok(())) + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll<Result<usize, io::Error>> { + Pin::new(&mut self.inner).poll_write_vectored(cx, bufs) } - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> { - Poll::Ready(Ok(())) + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() } } impl AsyncRead for ChildStdout { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll<io::Result<()>> { - // Safety: pipes support reading into uninitialized memory - unsafe { self.inner.poll_read(cx, buf) } + Pin::new(&mut self.inner).poll_read(cx, buf) } } impl AsyncRead for ChildStderr { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll<io::Result<()>> { - // Safety: pipes support reading into uninitialized memory - unsafe { self.inner.poll_read(cx, buf) } + Pin::new(&mut self.inner).poll_read(cx, buf) } } diff --git a/src/process/unix/mod.rs b/src/process/unix/mod.rs index 576fe6c..78c792c 100644 --- a/src/process/unix/mod.rs +++ b/src/process/unix/mod.rs @@ -21,23 +21,20 @@ //! processes in general aren't scalable (e.g. millions) so it shouldn't be that //! bad in theory... -pub(crate) mod driver; - pub(crate) mod orphan; use orphan::{OrphanQueue, OrphanQueueImpl, Wait}; mod reap; use reap::Reaper; -use crate::io::PollEvented; +use crate::io::{AsyncRead, AsyncWrite, PollEvented, ReadBuf}; use crate::process::kill::Kill; use crate::process::SpawnedChild; -use crate::signal::unix::driver::Handle as SignalHandle; +use crate::runtime::signal::Handle as SignalHandle; use crate::signal::unix::{signal, Signal, SignalKind}; use mio::event::Source; use mio::unix::SourceFd; -use once_cell::sync::Lazy; use std::fmt; use std::fs::File; use std::future::Future; @@ -64,25 +61,41 @@ impl Kill for StdChild { } } -static ORPHAN_QUEUE: Lazy<OrphanQueueImpl<StdChild>> = Lazy::new(OrphanQueueImpl::new); +cfg_not_has_const_mutex_new! { + fn get_orphan_queue() -> &'static OrphanQueueImpl<StdChild> { + use crate::util::once_cell::OnceCell; + + static ORPHAN_QUEUE: OnceCell<OrphanQueueImpl<StdChild>> = OnceCell::new(); + + ORPHAN_QUEUE.get(OrphanQueueImpl::new) + } +} + +cfg_has_const_mutex_new! { + fn get_orphan_queue() -> &'static OrphanQueueImpl<StdChild> { + static ORPHAN_QUEUE: OrphanQueueImpl<StdChild> = OrphanQueueImpl::new(); + + &ORPHAN_QUEUE + } +} pub(crate) struct GlobalOrphanQueue; impl fmt::Debug for GlobalOrphanQueue { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - ORPHAN_QUEUE.fmt(fmt) + get_orphan_queue().fmt(fmt) } } impl GlobalOrphanQueue { - fn reap_orphans(handle: &SignalHandle) { - ORPHAN_QUEUE.reap_orphans(handle) + pub(crate) fn reap_orphans(handle: &SignalHandle) { + get_orphan_queue().reap_orphans(handle) } } impl OrphanQueue<StdChild> for GlobalOrphanQueue { fn push_orphan(&self, orphan: StdChild) { - ORPHAN_QUEUE.push_orphan(orphan) + get_orphan_queue().push_orphan(orphan) } } @@ -143,7 +156,7 @@ impl Future for Child { #[derive(Debug)] pub(crate) struct Pipe { - // Actually a pipe and not a File. However, we are reusing `File` to get + // Actually a pipe is not a File. However, we are reusing `File` to get // close on drop. This is a similar trick as `mio`. fd: File, } @@ -169,6 +182,10 @@ impl<'a> io::Write for &'a Pipe { fn flush(&mut self) -> io::Result<()> { (&self.fd).flush() } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result<usize> { + (&self.fd).write_vectored(bufs) + } } impl AsRawFd for Pipe { @@ -177,8 +194,8 @@ impl AsRawFd for Pipe { } } -pub(crate) fn convert_to_stdio(io: PollEvented<Pipe>) -> io::Result<Stdio> { - let mut fd = io.into_inner()?.fd; +pub(crate) fn convert_to_stdio(io: ChildStdio) -> io::Result<Stdio> { + let mut fd = io.inner.into_inner()?.fd; // Ensure that the fd to be inherited is set to *blocking* mode, as this // is the default that virtually all programs expect to have. Those @@ -213,7 +230,62 @@ impl Source for Pipe { } } -pub(crate) type ChildStdio = PollEvented<Pipe>; +pub(crate) struct ChildStdio { + inner: PollEvented<Pipe>, +} + +impl fmt::Debug for ChildStdio { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(fmt) + } +} + +impl AsRawFd for ChildStdio { + fn as_raw_fd(&self) -> RawFd { + self.inner.as_raw_fd() + } +} + +impl AsyncWrite for ChildStdio { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll<io::Result<usize>> { + self.inner.poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> { + Poll::Ready(Ok(())) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll<Result<usize, io::Error>> { + self.inner.poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl AsyncRead for ChildStdio { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + // Safety: pipes support reading into uninitialized memory + unsafe { self.inner.poll_read(cx, buf) } + } +} fn set_nonblocking<T: AsRawFd>(fd: &mut T, nonblocking: bool) -> io::Result<()> { unsafe { @@ -238,7 +310,7 @@ fn set_nonblocking<T: AsRawFd>(fd: &mut T, nonblocking: bool) -> io::Result<()> Ok(()) } -pub(super) fn stdio<T>(io: T) -> io::Result<PollEvented<Pipe>> +pub(super) fn stdio<T>(io: T) -> io::Result<ChildStdio> where T: IntoRawFd, { @@ -246,5 +318,5 @@ where let mut pipe = Pipe::from(io); set_nonblocking(&mut pipe, true)?; - PollEvented::new(pipe) + PollEvented::new(pipe).map(|inner| ChildStdio { inner }) } diff --git a/src/process/unix/orphan.rs b/src/process/unix/orphan.rs index 1b0022c..3407196 100644 --- a/src/process/unix/orphan.rs +++ b/src/process/unix/orphan.rs @@ -1,5 +1,5 @@ use crate::loom::sync::{Mutex, MutexGuard}; -use crate::signal::unix::driver::Handle as SignalHandle; +use crate::runtime::signal::Handle as SignalHandle; use crate::signal::unix::{signal_with_handle, SignalKind}; use crate::sync::watch; use std::io; @@ -43,10 +43,21 @@ pub(crate) struct OrphanQueueImpl<T> { } impl<T> OrphanQueueImpl<T> { - pub(crate) fn new() -> Self { - Self { - sigchild: Mutex::new(None), - queue: Mutex::new(Vec::new()), + cfg_not_has_const_mutex_new! { + pub(crate) fn new() -> Self { + Self { + sigchild: Mutex::new(None), + queue: Mutex::new(Vec::new()), + } + } + } + + cfg_has_const_mutex_new! { + pub(crate) const fn new() -> Self { + Self { + sigchild: Mutex::const_new(None), + queue: Mutex::const_new(Vec::new()), + } } } @@ -120,8 +131,8 @@ where #[cfg(all(test, not(loom)))] pub(crate) mod test { use super::*; - use crate::io::driver::Driver as IoDriver; - use crate::signal::unix::driver::{Driver as SignalDriver, Handle as SignalHandle}; + use crate::runtime::io::Driver as IoDriver; + use crate::runtime::signal::{Driver as SignalDriver, Handle as SignalHandle}; use crate::sync::watch; use std::cell::{Cell, RefCell}; use std::io; @@ -280,9 +291,11 @@ pub(crate) mod test { drop(signal_guard); } + #[cfg_attr(miri, ignore)] // Miri does not support epoll. #[test] fn does_not_register_signal_if_queue_empty() { - let signal_driver = IoDriver::new().and_then(SignalDriver::new).unwrap(); + let (io_driver, io_handle) = IoDriver::new(1024).unwrap(); + let signal_driver = SignalDriver::new(io_driver, &io_handle).unwrap(); let handle = signal_driver.handle(); let orphanage = OrphanQueueImpl::new(); diff --git a/src/process/windows.rs b/src/process/windows.rs index 136d5b0..0e4c7dd 100644 --- a/src/process/windows.rs +++ b/src/process/windows.rs @@ -15,29 +15,31 @@ //! `RegisterWaitForSingleObject` and then wait on the other end of the oneshot //! from then on out. -use crate::io::PollEvented; +use crate::io::{blocking::Blocking, AsyncRead, AsyncWrite, ReadBuf}; use crate::process::kill::Kill; use crate::process::SpawnedChild; use crate::sync::oneshot; -use mio::windows::NamedPipe; use std::fmt; +use std::fs::File as StdFile; use std::future::Future; use std::io; -use std::os::windows::prelude::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle}; +use std::os::windows::prelude::{AsRawHandle, IntoRawHandle, RawHandle}; use std::pin::Pin; use std::process::Stdio; use std::process::{Child as StdChild, Command as StdCommand, ExitStatus}; -use std::ptr; -use std::task::Context; -use std::task::Poll; -use winapi::shared::minwindef::{DWORD, FALSE}; -use winapi::um::handleapi::{DuplicateHandle, INVALID_HANDLE_VALUE}; -use winapi::um::processthreadsapi::GetCurrentProcess; -use winapi::um::threadpoollegacyapiset::UnregisterWaitEx; -use winapi::um::winbase::{RegisterWaitForSingleObject, INFINITE}; -use winapi::um::winnt::{ - BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, PVOID, WT_EXECUTEINWAITTHREAD, WT_EXECUTEONLYONCE, +use std::sync::Arc; +use std::task::{Context, Poll}; + +use windows_sys::{ + Win32::Foundation::{ + DuplicateHandle, BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, INVALID_HANDLE_VALUE, + }, + Win32::System::Threading::{ + GetCurrentProcess, RegisterWaitForSingleObject, UnregisterWaitEx, WT_EXECUTEINWAITTHREAD, + WT_EXECUTEONLYONCE, + }, + Win32::System::WindowsProgramming::INFINITE, }; #[must_use = "futures do nothing unless polled"] @@ -119,11 +121,11 @@ impl Future for Child { } let (tx, rx) = oneshot::channel(); let ptr = Box::into_raw(Box::new(Some(tx))); - let mut wait_object = ptr::null_mut(); + let mut wait_object = 0; let rc = unsafe { RegisterWaitForSingleObject( &mut wait_object, - inner.child.as_raw_handle(), + inner.child.as_raw_handle() as _, Some(callback), ptr as *mut _, INFINITE, @@ -162,37 +164,106 @@ impl Drop for Waiting { } } -unsafe extern "system" fn callback(ptr: PVOID, _timer_fired: BOOLEAN) { +unsafe extern "system" fn callback(ptr: *mut std::ffi::c_void, _timer_fired: BOOLEAN) { let complete = &mut *(ptr as *mut Option<oneshot::Sender<()>>); let _ = complete.take().unwrap().send(()); } -pub(crate) type ChildStdio = PollEvented<NamedPipe>; +#[derive(Debug)] +struct ArcFile(Arc<StdFile>); + +impl io::Read for ArcFile { + fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> { + (&*self.0).read(bytes) + } +} + +impl io::Write for ArcFile { + fn write(&mut self, bytes: &[u8]) -> io::Result<usize> { + (&*self.0).write(bytes) + } + + fn flush(&mut self) -> io::Result<()> { + (&*self.0).flush() + } +} + +#[derive(Debug)] +pub(crate) struct ChildStdio { + // Used for accessing the raw handle, even if the io version is busy + raw: Arc<StdFile>, + // For doing I/O operations asynchronously + io: Blocking<ArcFile>, +} + +impl AsRawHandle for ChildStdio { + fn as_raw_handle(&self) -> RawHandle { + self.raw.as_raw_handle() + } +} -pub(super) fn stdio<T>(io: T) -> io::Result<PollEvented<NamedPipe>> +impl AsyncRead for ChildStdio { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + Pin::new(&mut self.io).poll_read(cx, buf) + } +} + +impl AsyncWrite for ChildStdio { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll<io::Result<usize>> { + Pin::new(&mut self.io).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> { + Pin::new(&mut self.io).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> { + Pin::new(&mut self.io).poll_shutdown(cx) + } +} + +pub(super) fn stdio<T>(io: T) -> io::Result<ChildStdio> where T: IntoRawHandle, { - let pipe = unsafe { NamedPipe::from_raw_handle(io.into_raw_handle()) }; - PollEvented::new(pipe) + use std::os::windows::prelude::FromRawHandle; + + let raw = Arc::new(unsafe { StdFile::from_raw_handle(io.into_raw_handle()) }); + let io = Blocking::new(ArcFile(raw.clone())); + Ok(ChildStdio { raw, io }) +} + +pub(crate) fn convert_to_stdio(child_stdio: ChildStdio) -> io::Result<Stdio> { + let ChildStdio { raw, io } = child_stdio; + drop(io); // Try to drop the Arc count here + + Arc::try_unwrap(raw) + .or_else(|raw| duplicate_handle(&*raw)) + .map(Stdio::from) } -pub(crate) fn convert_to_stdio(io: PollEvented<NamedPipe>) -> io::Result<Stdio> { - let named_pipe = io.into_inner()?; +fn duplicate_handle<T: AsRawHandle>(io: &T) -> io::Result<StdFile> { + use std::os::windows::prelude::FromRawHandle; - // Mio does not implement `IntoRawHandle` for `NamedPipe`, so we'll manually - // duplicate the handle here... unsafe { let mut dup_handle = INVALID_HANDLE_VALUE; let cur_proc = GetCurrentProcess(); let status = DuplicateHandle( cur_proc, - named_pipe.as_raw_handle(), + io.as_raw_handle() as _, cur_proc, &mut dup_handle, - 0 as DWORD, - FALSE, + 0, + 0, DUPLICATE_SAME_ACCESS, ); @@ -200,6 +271,6 @@ pub(crate) fn convert_to_stdio(io: PollEvented<NamedPipe>) -> io::Result<Stdio> return Err(io::Error::last_os_error()); } - Ok(Stdio::from_raw_handle(dup_handle)) + Ok(StdFile::from_raw_handle(dup_handle as _)) } } diff --git a/src/runtime/basic_scheduler.rs b/src/runtime/basic_scheduler.rs deleted file mode 100644 index 872d0d5..0000000 --- a/src/runtime/basic_scheduler.rs +++ /dev/null @@ -1,519 +0,0 @@ -use crate::future::poll_fn; -use crate::loom::sync::atomic::AtomicBool; -use crate::loom::sync::Mutex; -use crate::park::{Park, Unpark}; -use crate::runtime::context::EnterGuard; -use crate::runtime::stats::{RuntimeStats, WorkerStatsBatcher}; -use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task}; -use crate::runtime::Callback; -use crate::sync::notify::Notify; -use crate::util::{waker_ref, Wake, WakerRef}; - -use std::cell::RefCell; -use std::collections::VecDeque; -use std::fmt; -use std::future::Future; -use std::sync::atomic::Ordering::{AcqRel, Release}; -use std::sync::Arc; -use std::task::Poll::{Pending, Ready}; -use std::time::Duration; - -/// Executes tasks on the current thread -pub(crate) struct BasicScheduler<P: Park> { - /// Inner state guarded by a mutex that is shared - /// between all `block_on` calls. - inner: Mutex<Option<Inner<P>>>, - - /// Notifier for waking up other threads to steal the - /// parker. - notify: Notify, - - /// Sendable task spawner - spawner: Spawner, - - /// This is usually None, but right before dropping the BasicScheduler, it - /// is changed to `Some` with the context being the runtime's own context. - /// This ensures that any tasks dropped in the `BasicScheduler`s destructor - /// run in that runtime's context. - context_guard: Option<EnterGuard>, -} - -/// The inner scheduler that owns the task queue and the main parker P. -struct Inner<P: Park> { - /// Scheduler run queue - /// - /// When the scheduler is executed, the queue is removed from `self` and - /// moved into `Context`. - /// - /// This indirection is to allow `BasicScheduler` to be `Send`. - tasks: Option<Tasks>, - - /// Sendable task spawner - spawner: Spawner, - - /// Current tick - tick: u8, - - /// Thread park handle - park: P, - - /// Callback for a worker parking itself - before_park: Option<Callback>, - /// Callback for a worker unparking itself - after_unpark: Option<Callback>, - - /// Stats batcher - stats: WorkerStatsBatcher, -} - -#[derive(Clone)] -pub(crate) struct Spawner { - shared: Arc<Shared>, -} - -struct Tasks { - /// Local run queue. - /// - /// Tasks notified from the current thread are pushed into this queue. - queue: VecDeque<task::Notified<Arc<Shared>>>, -} - -/// A remote scheduler entry. -/// -/// These are filled in by remote threads sending instructions to the scheduler. -enum RemoteMsg { - /// A remote thread wants to spawn a task. - Schedule(task::Notified<Arc<Shared>>), -} - -// Safety: Used correctly, the task header is "thread safe". Ultimately the task -// is owned by the current thread executor, for which this instruction is being -// sent. -unsafe impl Send for RemoteMsg {} - -/// Scheduler state shared between threads. -struct Shared { - /// Remote run queue. None if the `Runtime` has been dropped. - queue: Mutex<Option<VecDeque<RemoteMsg>>>, - - /// Collection of all active tasks spawned onto this executor. - owned: OwnedTasks<Arc<Shared>>, - - /// Unpark the blocked thread. - unpark: Box<dyn Unpark>, - - /// Indicates whether the blocked on thread was woken. - woken: AtomicBool, - - /// Keeps track of various runtime stats. - stats: RuntimeStats, -} - -/// Thread-local context. -struct Context { - /// Shared scheduler state - shared: Arc<Shared>, - - /// Local queue - tasks: RefCell<Tasks>, -} - -/// Initial queue capacity. -const INITIAL_CAPACITY: usize = 64; - -/// Max number of tasks to poll per tick. -#[cfg(loom)] -const MAX_TASKS_PER_TICK: usize = 4; -#[cfg(not(loom))] -const MAX_TASKS_PER_TICK: usize = 61; - -/// How often to check the remote queue first. -const REMOTE_FIRST_INTERVAL: u8 = 31; - -// Tracks the current BasicScheduler. -scoped_thread_local!(static CURRENT: Context); - -impl<P: Park> BasicScheduler<P> { - pub(crate) fn new( - park: P, - before_park: Option<Callback>, - after_unpark: Option<Callback>, - ) -> BasicScheduler<P> { - let unpark = Box::new(park.unpark()); - - let spawner = Spawner { - shared: Arc::new(Shared { - queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))), - owned: OwnedTasks::new(), - unpark: unpark as Box<dyn Unpark>, - woken: AtomicBool::new(false), - stats: RuntimeStats::new(1), - }), - }; - - let inner = Mutex::new(Some(Inner { - tasks: Some(Tasks { - queue: VecDeque::with_capacity(INITIAL_CAPACITY), - }), - spawner: spawner.clone(), - tick: 0, - park, - before_park, - after_unpark, - stats: WorkerStatsBatcher::new(0), - })); - - BasicScheduler { - inner, - notify: Notify::new(), - spawner, - context_guard: None, - } - } - - pub(crate) fn spawner(&self) -> &Spawner { - &self.spawner - } - - pub(crate) fn block_on<F: Future>(&self, future: F) -> F::Output { - pin!(future); - - // Attempt to steal the dedicated parker and block_on the future if we can there, - // otherwise, lets select on a notification that the parker is available - // or the future is complete. - loop { - if let Some(inner) = &mut self.take_inner() { - return inner.block_on(future); - } else { - let mut enter = crate::runtime::enter(false); - - let notified = self.notify.notified(); - pin!(notified); - - if let Some(out) = enter - .block_on(poll_fn(|cx| { - if notified.as_mut().poll(cx).is_ready() { - return Ready(None); - } - - if let Ready(out) = future.as_mut().poll(cx) { - return Ready(Some(out)); - } - - Pending - })) - .expect("Failed to `Enter::block_on`") - { - return out; - } - } - } - } - - fn take_inner(&self) -> Option<InnerGuard<'_, P>> { - let inner = self.inner.lock().take()?; - - Some(InnerGuard { - inner: Some(inner), - basic_scheduler: self, - }) - } - - pub(super) fn set_context_guard(&mut self, guard: EnterGuard) { - self.context_guard = Some(guard); - } -} - -impl<P: Park> Inner<P> { - /// Blocks on the provided future and drives the runtime's driver. - fn block_on<F: Future>(&mut self, future: F) -> F::Output { - enter(self, |scheduler, context| { - let _enter = crate::runtime::enter(false); - let waker = scheduler.spawner.waker_ref(); - let mut cx = std::task::Context::from_waker(&waker); - - pin!(future); - - 'outer: loop { - if scheduler.spawner.reset_woken() { - scheduler.stats.incr_poll_count(); - if let Ready(v) = crate::coop::budget(|| future.as_mut().poll(&mut cx)) { - return v; - } - } - - for _ in 0..MAX_TASKS_PER_TICK { - // Get and increment the current tick - let tick = scheduler.tick; - scheduler.tick = scheduler.tick.wrapping_add(1); - - let entry = if tick % REMOTE_FIRST_INTERVAL == 0 { - scheduler.spawner.pop().or_else(|| { - context - .tasks - .borrow_mut() - .queue - .pop_front() - .map(RemoteMsg::Schedule) - }) - } else { - context - .tasks - .borrow_mut() - .queue - .pop_front() - .map(RemoteMsg::Schedule) - .or_else(|| scheduler.spawner.pop()) - }; - - let entry = match entry { - Some(entry) => entry, - None => { - if let Some(f) = &scheduler.before_park { - f(); - } - // This check will fail if `before_park` spawns a task for us to run - // instead of parking the thread - if context.tasks.borrow_mut().queue.is_empty() { - // Park until the thread is signaled - scheduler.stats.about_to_park(); - scheduler.stats.submit(&scheduler.spawner.shared.stats); - scheduler.park.park().expect("failed to park"); - scheduler.stats.returned_from_park(); - } - if let Some(f) = &scheduler.after_unpark { - f(); - } - - // Try polling the `block_on` future next - continue 'outer; - } - }; - - match entry { - RemoteMsg::Schedule(task) => { - scheduler.stats.incr_poll_count(); - let task = context.shared.owned.assert_owner(task); - crate::coop::budget(|| task.run()) - } - } - } - - // Yield to the park, this drives the timer and pulls any pending - // I/O events. - scheduler.stats.submit(&scheduler.spawner.shared.stats); - scheduler - .park - .park_timeout(Duration::from_millis(0)) - .expect("failed to park"); - } - }) - } -} - -/// Enters the scheduler context. This sets the queue and other necessary -/// scheduler state in the thread-local. -fn enter<F, R, P>(scheduler: &mut Inner<P>, f: F) -> R -where - F: FnOnce(&mut Inner<P>, &Context) -> R, - P: Park, -{ - // Ensures the run queue is placed back in the `BasicScheduler` instance - // once `block_on` returns.` - struct Guard<'a, P: Park> { - context: Option<Context>, - scheduler: &'a mut Inner<P>, - } - - impl<P: Park> Drop for Guard<'_, P> { - fn drop(&mut self) { - let Context { tasks, .. } = self.context.take().expect("context missing"); - self.scheduler.tasks = Some(tasks.into_inner()); - } - } - - // Remove `tasks` from `self` and place it in a `Context`. - let tasks = scheduler.tasks.take().expect("invalid state"); - - let guard = Guard { - context: Some(Context { - shared: scheduler.spawner.shared.clone(), - tasks: RefCell::new(tasks), - }), - scheduler, - }; - - let context = guard.context.as_ref().unwrap(); - let scheduler = &mut *guard.scheduler; - - CURRENT.set(context, || f(scheduler, context)) -} - -impl<P: Park> Drop for BasicScheduler<P> { - fn drop(&mut self) { - // Avoid a double panic if we are currently panicking and - // the lock may be poisoned. - - let mut inner = match self.inner.lock().take() { - Some(inner) => inner, - None if std::thread::panicking() => return, - None => panic!("Oh no! We never placed the Inner state back, this is a bug!"), - }; - - enter(&mut inner, |scheduler, context| { - // Drain the OwnedTasks collection. This call also closes the - // collection, ensuring that no tasks are ever pushed after this - // call returns. - context.shared.owned.close_and_shutdown_all(); - - // Drain local queue - // We already shut down every task, so we just need to drop the task. - for task in context.tasks.borrow_mut().queue.drain(..) { - drop(task); - } - - // Drain remote queue and set it to None - let remote_queue = scheduler.spawner.shared.queue.lock().take(); - - // Using `Option::take` to replace the shared queue with `None`. - // We already shut down every task, so we just need to drop the task. - if let Some(remote_queue) = remote_queue { - for entry in remote_queue { - match entry { - RemoteMsg::Schedule(task) => { - drop(task); - } - } - } - } - - assert!(context.shared.owned.is_empty()); - }); - } -} - -impl<P: Park> fmt::Debug for BasicScheduler<P> { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("BasicScheduler").finish() - } -} - -// ===== impl Spawner ===== - -impl Spawner { - /// Spawns a future onto the basic scheduler - pub(crate) fn spawn<F>(&self, future: F) -> JoinHandle<F::Output> - where - F: crate::future::Future + Send + 'static, - F::Output: Send + 'static, - { - let (handle, notified) = self.shared.owned.bind(future, self.shared.clone()); - - if let Some(notified) = notified { - self.shared.schedule(notified); - } - - handle - } - - pub(crate) fn stats(&self) -> &RuntimeStats { - &self.shared.stats - } - - fn pop(&self) -> Option<RemoteMsg> { - match self.shared.queue.lock().as_mut() { - Some(queue) => queue.pop_front(), - None => None, - } - } - - fn waker_ref(&self) -> WakerRef<'_> { - // Set woken to true when enter block_on, ensure outer future - // be polled for the first time when enter loop - self.shared.woken.store(true, Release); - waker_ref(&self.shared) - } - - // reset woken to false and return original value - pub(crate) fn reset_woken(&self) -> bool { - self.shared.woken.swap(false, AcqRel) - } -} - -impl fmt::Debug for Spawner { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Spawner").finish() - } -} - -// ===== impl Shared ===== - -impl Schedule for Arc<Shared> { - fn release(&self, task: &Task<Self>) -> Option<Task<Self>> { - self.owned.remove(task) - } - - fn schedule(&self, task: task::Notified<Self>) { - CURRENT.with(|maybe_cx| match maybe_cx { - Some(cx) if Arc::ptr_eq(self, &cx.shared) => { - cx.tasks.borrow_mut().queue.push_back(task); - } - _ => { - // If the queue is None, then the runtime has shut down. We - // don't need to do anything with the notification in that case. - let mut guard = self.queue.lock(); - if let Some(queue) = guard.as_mut() { - queue.push_back(RemoteMsg::Schedule(task)); - drop(guard); - self.unpark.unpark(); - } - } - }); - } -} - -impl Wake for Shared { - fn wake(self: Arc<Self>) { - Wake::wake_by_ref(&self) - } - - /// Wake by reference - fn wake_by_ref(arc_self: &Arc<Self>) { - arc_self.woken.store(true, Release); - arc_self.unpark.unpark(); - } -} - -// ===== InnerGuard ===== - -/// Used to ensure we always place the Inner value -/// back into its slot in `BasicScheduler`, even if the -/// future panics. -struct InnerGuard<'a, P: Park> { - inner: Option<Inner<P>>, - basic_scheduler: &'a BasicScheduler<P>, -} - -impl<P: Park> InnerGuard<'_, P> { - fn block_on<F: Future>(&mut self, future: F) -> F::Output { - // The only time inner gets set to `None` is if we have dropped - // already so this unwrap is safe. - self.inner.as_mut().unwrap().block_on(future) - } -} - -impl<P: Park> Drop for InnerGuard<'_, P> { - fn drop(&mut self) { - if let Some(scheduler) = self.inner.take() { - let mut lock = self.basic_scheduler.inner.lock(); - - // Replace old scheduler back into the state to allow - // other threads to pick it up and drive it. - lock.replace(scheduler); - - // Wake up other possible threads that could steal - // the dedicated parker P. - self.basic_scheduler.notify.notify_one() - } - } -} diff --git a/src/runtime/blocking/mod.rs b/src/runtime/blocking/mod.rs index 670ec3a..c42924b 100644 --- a/src/runtime/blocking/mod.rs +++ b/src/runtime/blocking/mod.rs @@ -6,10 +6,17 @@ mod pool; pub(crate) use pool::{spawn_blocking, BlockingPool, Spawner}; +cfg_fs! { + pub(crate) use pool::spawn_mandatory_blocking; +} + +cfg_trace! { + pub(crate) use pool::Mandatory; +} + mod schedule; mod shutdown; mod task; -pub(crate) use schedule::NoopSchedule; pub(crate) use task::BlockingTask; use crate::runtime::Builder; @@ -17,28 +24,3 @@ use crate::runtime::Builder; pub(crate) fn create_blocking_pool(builder: &Builder, thread_cap: usize) -> BlockingPool { BlockingPool::new(builder, thread_cap) } - -/* -cfg_not_blocking_impl! { - use crate::runtime::Builder; - use std::time::Duration; - - #[derive(Debug, Clone)] - pub(crate) struct BlockingPool {} - - pub(crate) use BlockingPool as Spawner; - - pub(crate) fn create_blocking_pool(_builder: &Builder, _thread_cap: usize) -> BlockingPool { - BlockingPool {} - } - - impl BlockingPool { - pub(crate) fn spawner(&self) -> &BlockingPool { - self - } - - pub(crate) fn shutdown(&mut self, _duration: Option<Duration>) { - } - } -} -*/ diff --git a/src/runtime/blocking/pool.rs b/src/runtime/blocking/pool.rs index 77ab495..e9f6b66 100644 --- a/src/runtime/blocking/pool.rs +++ b/src/runtime/blocking/pool.rs @@ -2,15 +2,16 @@ use crate::loom::sync::{Arc, Condvar, Mutex}; use crate::loom::thread; -use crate::runtime::blocking::schedule::NoopSchedule; -use crate::runtime::blocking::shutdown; +use crate::runtime::blocking::schedule::BlockingSchedule; +use crate::runtime::blocking::{shutdown, BlockingTask}; use crate::runtime::builder::ThreadNameFn; -use crate::runtime::context; use crate::runtime::task::{self, JoinHandle}; use crate::runtime::{Builder, Callback, Handle}; use std::collections::{HashMap, VecDeque}; use std::fmt; +use std::io; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; pub(crate) struct BlockingPool { @@ -23,6 +24,53 @@ pub(crate) struct Spawner { inner: Arc<Inner>, } +#[derive(Default)] +pub(crate) struct SpawnerMetrics { + num_threads: AtomicUsize, + num_idle_threads: AtomicUsize, + queue_depth: AtomicUsize, +} + +impl SpawnerMetrics { + fn num_threads(&self) -> usize { + self.num_threads.load(Ordering::Relaxed) + } + + fn num_idle_threads(&self) -> usize { + self.num_idle_threads.load(Ordering::Relaxed) + } + + cfg_metrics! { + fn queue_depth(&self) -> usize { + self.queue_depth.load(Ordering::Relaxed) + } + } + + fn inc_num_threads(&self) { + self.num_threads.fetch_add(1, Ordering::Relaxed); + } + + fn dec_num_threads(&self) { + self.num_threads.fetch_sub(1, Ordering::Relaxed); + } + + fn inc_num_idle_threads(&self) { + self.num_idle_threads.fetch_add(1, Ordering::Relaxed); + } + + fn dec_num_idle_threads(&self) -> usize { + self.num_idle_threads.fetch_sub(1, Ordering::Relaxed) + } + + fn inc_queue_depth(&self) { + self.queue_depth.fetch_add(1, Ordering::Relaxed); + } + + fn dec_queue_depth(&self) { + self.queue_depth.fetch_sub(1, Ordering::Relaxed); + } +} + struct Inner { /// State shared between worker threads. shared: Mutex<Shared>, @@ -47,12 +95,13 @@ struct Inner { // Customizable wait timeout. keep_alive: Duration, + + // Metrics about the pool. + metrics: SpawnerMetrics, } struct Shared { queue: VecDeque<Task>, - num_th: usize, - num_idle: u32, num_notify: u32, shutdown: bool, shutdown_tx: Option<shutdown::Sender>, @@ -70,20 +119,89 @@ struct Shared { worker_thread_index: usize, } -type Task = task::UnownedTask<NoopSchedule>; +pub(crate) struct Task { + task: task::UnownedTask<BlockingSchedule>, + mandatory: Mandatory, +} + +#[derive(PartialEq, Eq)] +pub(crate) enum Mandatory { + #[cfg_attr(not(fs), allow(dead_code))] + Mandatory, + NonMandatory, +} + +pub(crate) enum SpawnError { + /// Pool is shutting down and the task was not scheduled + ShuttingDown, + /// There are no worker threads available to take the task + /// and the OS failed to spawn a new one + NoThreads(io::Error), +} + +impl From<SpawnError> for io::Error { + fn from(e: SpawnError) -> Self { + match e { + SpawnError::ShuttingDown => { + io::Error::new(io::ErrorKind::Other, "blocking pool shutting down") + } + SpawnError::NoThreads(e) => e, + } + } +} + +impl Task { + pub(crate) fn new(task: task::UnownedTask<BlockingSchedule>, mandatory: Mandatory) -> Task { + Task { task, mandatory } + } + + fn run(self) { + self.task.run(); + } + + fn shutdown_or_run_if_mandatory(self) { + match self.mandatory { + Mandatory::NonMandatory => self.task.shutdown(), + Mandatory::Mandatory => self.task.run(), + } + } +} const KEEP_ALIVE: Duration = Duration::from_secs(10); /// Runs the provided function on an executor dedicated to blocking operations. +/// Tasks will be scheduled as non-mandatory, meaning they may not get executed +/// in case of runtime shutdown. +#[track_caller] +#[cfg_attr(tokio_wasi, allow(dead_code))] pub(crate) fn spawn_blocking<F, R>(func: F) -> JoinHandle<R> where F: FnOnce() -> R + Send + 'static, R: Send + 'static, { - let rt = context::current(); + let rt = Handle::current(); rt.spawn_blocking(func) } +cfg_fs! { + #[cfg_attr(any( + all(loom, not(test)), // the function is covered by loom tests + test + ), allow(dead_code))] + /// Runs the provided function on an executor dedicated to blocking + /// operations. Tasks will be scheduled as mandatory, meaning they are + /// guaranteed to run unless a shutdown is already taking place. In case a + /// shutdown is already taking place, `None` will be returned. + pub(crate) fn spawn_mandatory_blocking<F, R>(func: F) -> Option<JoinHandle<R>> + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + let rt = Handle::current(); + rt.inner.blocking_spawner().spawn_mandatory_blocking(&rt, func) + } +} + // ===== impl BlockingPool ===== impl BlockingPool { @@ -96,8 +214,6 @@ impl BlockingPool { inner: Arc::new(Inner { shared: Mutex::new(Shared { queue: VecDeque::new(), - num_th: 0, - num_idle: 0, num_notify: 0, shutdown: false, shutdown_tx: Some(shutdown_tx), @@ -112,6 +228,7 @@ impl BlockingPool { before_stop: builder.before_stop.clone(), thread_cap, keep_alive, + metrics: Default::default(), }), }, shutdown_rx, @@ -171,53 +288,162 @@ impl fmt::Debug for BlockingPool { // ===== impl Spawner ===== impl Spawner { - pub(crate) fn spawn(&self, task: Task, rt: &Handle) -> Result<(), ()> { - let shutdown_tx = { - let mut shared = self.inner.shared.lock(); - - if shared.shutdown { - // Shutdown the task - task.shutdown(); - - // no need to even push this task; it would never get picked up - return Err(()); + #[track_caller] + pub(crate) fn spawn_blocking<F, R>(&self, rt: &Handle, func: F) -> JoinHandle<R> + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + let (join_handle, spawn_result) = + if cfg!(debug_assertions) && std::mem::size_of::<F>() > 2048 { + self.spawn_blocking_inner(Box::new(func), Mandatory::NonMandatory, None, rt) + } else { + self.spawn_blocking_inner(func, Mandatory::NonMandatory, None, rt) + }; + + match spawn_result { + Ok(()) => join_handle, + // Compat: do not panic here, return the join_handle even though it will never resolve + Err(SpawnError::ShuttingDown) => join_handle, + Err(SpawnError::NoThreads(e)) => { + panic!("OS can't spawn worker thread: {}", e) } + } + } - shared.queue.push_back(task); - - if shared.num_idle == 0 { - // No threads are able to process the task. - - if shared.num_th == self.inner.thread_cap { - // At max number of threads - None - } else { - shared.num_th += 1; - assert!(shared.shutdown_tx.is_some()); - shared.shutdown_tx.clone() - } + cfg_fs! { + #[track_caller] + #[cfg_attr(any( + all(loom, not(test)), // the function is covered by loom tests + test + ), allow(dead_code))] + pub(crate) fn spawn_mandatory_blocking<F, R>(&self, rt: &Handle, func: F) -> Option<JoinHandle<R>> + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + let (join_handle, spawn_result) = if cfg!(debug_assertions) && std::mem::size_of::<F>() > 2048 { + self.spawn_blocking_inner( + Box::new(func), + Mandatory::Mandatory, + None, + rt, + ) + } else { + self.spawn_blocking_inner( + func, + Mandatory::Mandatory, + None, + rt, + ) + }; + + if spawn_result.is_ok() { + Some(join_handle) } else { - // Notify an idle worker thread. The notification counter - // is used to count the needed amount of notifications - // exactly. Thread libraries may generate spurious - // wakeups, this counter is used to keep us in a - // consistent state. - shared.num_idle -= 1; - shared.num_notify += 1; - self.inner.condvar.notify_one(); None } + } + } + + #[track_caller] + pub(crate) fn spawn_blocking_inner<F, R>( + &self, + func: F, + is_mandatory: Mandatory, + name: Option<&str>, + rt: &Handle, + ) -> (JoinHandle<R>, Result<(), SpawnError>) + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + let fut = BlockingTask::new(func); + let id = task::Id::next(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let fut = { + use tracing::Instrument; + let location = std::panic::Location::caller(); + let span = tracing::trace_span!( + target: "tokio::task::blocking", + "runtime.spawn", + kind = %"blocking", + task.name = %name.unwrap_or_default(), + task.id = id.as_u64(), + "fn" = %std::any::type_name::<F>(), + spawn.location = %format_args!("{}:{}:{}", location.file(), location.line(), location.column()), + ); + fut.instrument(span) }; - if let Some(shutdown_tx) = shutdown_tx { - let mut shared = self.inner.shared.lock(); + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let _ = name; - let id = shared.worker_thread_index; - shared.worker_thread_index += 1; + let (task, handle) = task::unowned(fut, BlockingSchedule::new(rt), id); - let handle = self.spawn_thread(shutdown_tx, rt, id); + let spawned = self.spawn_task(Task::new(task, is_mandatory), rt); + (handle, spawned) + } - shared.worker_threads.insert(id, handle); + fn spawn_task(&self, task: Task, rt: &Handle) -> Result<(), SpawnError> { + let mut shared = self.inner.shared.lock(); + + if shared.shutdown { + // Shutdown the task: it's fine to shutdown this task (even if + // mandatory) because it was scheduled after the shutdown of the + // runtime began. + task.task.shutdown(); + + // no need to even push this task; it would never get picked up + return Err(SpawnError::ShuttingDown); + } + + shared.queue.push_back(task); + self.inner.metrics.inc_queue_depth(); + + if self.inner.metrics.num_idle_threads() == 0 { + // No threads are able to process the task. + + if self.inner.metrics.num_threads() == self.inner.thread_cap { + // At max number of threads + } else { + assert!(shared.shutdown_tx.is_some()); + let shutdown_tx = shared.shutdown_tx.clone(); + + if let Some(shutdown_tx) = shutdown_tx { + let id = shared.worker_thread_index; + + match self.spawn_thread(shutdown_tx, rt, id) { + Ok(handle) => { + self.inner.metrics.inc_num_threads(); + shared.worker_thread_index += 1; + shared.worker_threads.insert(id, handle); + } + Err(ref e) + if is_temporary_os_thread_error(e) + && self.inner.metrics.num_threads() > 0 => + { + // OS temporarily failed to spawn a new thread. + // The task will be picked up eventually by a currently + // busy thread. + } + Err(e) => { + // The OS refused to spawn the thread and there is no thread + // to pick up the task that has just been pushed to the queue. + return Err(SpawnError::NoThreads(e)); + } + } + } + } + } else { + // Notify an idle worker thread. The notification counter + // is used to count the needed amount of notifications + // exactly. Thread libraries may generate spurious + // wakeups, this counter is used to keep us in a + // consistent state. + self.inner.metrics.dec_num_idle_threads(); + shared.num_notify += 1; + self.inner.condvar.notify_one(); } Ok(()) @@ -228,7 +454,7 @@ impl Spawner { shutdown_tx: shutdown::Sender, rt: &Handle, id: usize, - ) -> thread::JoinHandle<()> { + ) -> std::io::Result<thread::JoinHandle<()>> { let mut builder = thread::Builder::new().name((self.inner.thread_name)()); if let Some(stack_size) = self.inner.stack_size { @@ -237,17 +463,37 @@ impl Spawner { let rt = rt.clone(); - builder - .spawn(move || { - // Only the reference should be moved into the closure - let _enter = crate::runtime::context::enter(rt.clone()); - rt.blocking_spawner.inner.run(id); - drop(shutdown_tx); - }) - .unwrap() + builder.spawn(move || { + // Only the reference should be moved into the closure + let _enter = rt.enter(); + rt.inner.blocking_spawner().inner.run(id); + drop(shutdown_tx); + }) + } +} + +cfg_metrics! { + impl Spawner { + pub(crate) fn num_threads(&self) -> usize { + self.inner.metrics.num_threads() + } + + pub(crate) fn num_idle_threads(&self) -> usize { + self.inner.metrics.num_idle_threads() + } + + pub(crate) fn queue_depth(&self) -> usize { + self.inner.metrics.queue_depth() + } } } +// Tells whether the error when spawning a thread is temporary. +#[inline] +fn is_temporary_os_thread_error(error: &std::io::Error) -> bool { + matches!(error.kind(), std::io::ErrorKind::WouldBlock) +} + impl Inner { fn run(&self, worker_thread_id: usize) { if let Some(f) = &self.after_start { @@ -260,6 +506,7 @@ impl Inner { 'main: loop { // BUSY while let Some(task) = shared.queue.pop_front() { + self.metrics.dec_queue_depth(); drop(shared); task.run(); @@ -267,7 +514,7 @@ impl Inner { } // IDLE - shared.num_idle += 1; + self.metrics.inc_num_idle_threads(); while !shared.shutdown { let lock_result = self.condvar.wait_timeout(shared, self.keep_alive).unwrap(); @@ -301,8 +548,10 @@ impl Inner { if shared.shutdown { // Drain the queue while let Some(task) = shared.queue.pop_front() { + self.metrics.dec_queue_depth(); drop(shared); - task.shutdown(); + + task.shutdown_or_run_if_mandatory(); shared = self.shared.lock(); } @@ -310,7 +559,7 @@ impl Inner { // Work was produced, and we "took" it (by decrementing num_notify). // This means that num_idle was decremented once for our wakeup. // But, since we are exiting, we need to "undo" that, as we'll stay idle. - shared.num_idle += 1; + self.metrics.inc_num_idle_threads(); // NOTE: Technically we should also do num_notify++ and notify again, // but since we're shutting down anyway, that won't be necessary. break; @@ -318,17 +567,17 @@ impl Inner { } // Thread exit - shared.num_th -= 1; + self.metrics.dec_num_threads(); // num_idle should now be tracked exactly, panic // with a descriptive message if it is not the // case. - shared.num_idle = shared - .num_idle - .checked_sub(1) - .expect("num_idle underflowed on thread exit"); + let prev_idle = self.metrics.dec_num_idle_threads(); + if prev_idle < self.metrics.num_idle_threads() { + panic!("num_idle_threads underflowed on thread exit") + } - if shared.shutdown && shared.num_th == 0 { + if shared.shutdown && self.metrics.num_threads() == 0 { self.condvar.notify_one(); } diff --git a/src/runtime/blocking/schedule.rs b/src/runtime/blocking/schedule.rs index 5425224..edf775b 100644 --- a/src/runtime/blocking/schedule.rs +++ b/src/runtime/blocking/schedule.rs @@ -1,15 +1,52 @@ +#[cfg(feature = "test-util")] +use crate::runtime::scheduler; use crate::runtime::task::{self, Task}; +use crate::runtime::Handle; -/// `task::Schedule` implementation that does nothing. This is unique to the -/// blocking scheduler as tasks scheduled are not really futures but blocking -/// operations. +/// `task::Schedule` implementation that does nothing (except some bookkeeping +/// in test-util builds). This is unique to the blocking scheduler as tasks +/// scheduled are not really futures but blocking operations. /// /// We avoid storing the task by forgetting it in `bind` and re-materializing it -/// in `release. -pub(crate) struct NoopSchedule; +/// in `release`. +pub(crate) struct BlockingSchedule { + #[cfg(feature = "test-util")] + handle: Handle, +} + +impl BlockingSchedule { + #[cfg_attr(not(feature = "test-util"), allow(unused_variables))] + pub(crate) fn new(handle: &Handle) -> Self { + #[cfg(feature = "test-util")] + { + match &handle.inner { + scheduler::Handle::CurrentThread(handle) => { + handle.driver.clock.inhibit_auto_advance(); + } + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Handle::MultiThread(_) => {} + } + } + BlockingSchedule { + #[cfg(feature = "test-util")] + handle: handle.clone(), + } + } +} -impl task::Schedule for NoopSchedule { +impl task::Schedule for BlockingSchedule { fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> { + #[cfg(feature = "test-util")] + { + match &self.handle.inner { + scheduler::Handle::CurrentThread(handle) => { + handle.driver.clock.allow_auto_advance(); + handle.driver.unpark(); + } + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Handle::MultiThread(_) => {} + } + } None } diff --git a/src/runtime/blocking/shutdown.rs b/src/runtime/blocking/shutdown.rs index e6f4674..fe5abae 100644 --- a/src/runtime/blocking/shutdown.rs +++ b/src/runtime/blocking/shutdown.rs @@ -35,13 +35,13 @@ impl Receiver { /// /// If the timeout has elapsed, it returns `false`, otherwise it returns `true`. pub(crate) fn wait(&mut self, timeout: Option<Duration>) -> bool { - use crate::runtime::enter::try_enter; + use crate::runtime::context::try_enter_blocking_region; if timeout == Some(Duration::from_nanos(0)) { return false; } - let mut e = match try_enter(false) { + let mut e = match try_enter_blocking_region() { Some(enter) => enter, _ => { if std::thread::panicking() { diff --git a/src/runtime/blocking/task.rs b/src/runtime/blocking/task.rs index 0b7803a..c446175 100644 --- a/src/runtime/blocking/task.rs +++ b/src/runtime/blocking/task.rs @@ -37,7 +37,7 @@ where // currently goes through Task::poll(), and so is subject to budgeting. That isn't really // what we want; a blocking task may itself want to run tasks (it might be a Worker!), so // we want it to start without any budgeting. - crate::coop::stop(); + crate::runtime::coop::stop(); Poll::Ready(func()) } diff --git a/src/runtime/builder.rs b/src/runtime/builder.rs index 91c365f..ea0df2e 100644 --- a/src/runtime/builder.rs +++ b/src/runtime/builder.rs @@ -1,5 +1,6 @@ use crate::runtime::handle::Handle; -use crate::runtime::{blocking, driver, Callback, Runtime, Spawner}; +use crate::runtime::{blocking, driver, Callback, Runtime}; +use crate::util::rand::{RngSeed, RngSeedGenerator}; use std::fmt; use std::io; @@ -43,6 +44,7 @@ pub struct Builder { /// Whether or not to enable the I/O driver enable_io: bool, + nevents: usize, /// Whether or not to enable the time driver enable_time: bool, @@ -78,13 +80,112 @@ pub struct Builder { /// Customizable keep alive timeout for BlockingPool pub(super) keep_alive: Option<Duration>, + + /// How many ticks before pulling a task from the global/remote queue? + pub(super) global_queue_interval: u32, + + /// How many ticks before yielding to the driver for timer and I/O events? + pub(super) event_interval: u32, + + /// When true, the multi-threade scheduler LIFO slot should not be used. + /// + /// This option should only be exposed as unstable. + pub(super) disable_lifo_slot: bool, + + /// Specify a random number generator seed to provide deterministic results + pub(super) seed_generator: RngSeedGenerator, + + #[cfg(tokio_unstable)] + pub(super) unhandled_panic: UnhandledPanic, +} + +cfg_unstable! { + /// How the runtime should respond to unhandled panics. + /// + /// Instances of `UnhandledPanic` are passed to `Builder::unhandled_panic` + /// to configure the runtime behavior when a spawned task panics. + /// + /// See [`Builder::unhandled_panic`] for more details. + #[derive(Debug, Clone)] + #[non_exhaustive] + pub enum UnhandledPanic { + /// The runtime should ignore panics on spawned tasks. + /// + /// The panic is forwarded to the task's [`JoinHandle`] and all spawned + /// tasks continue running normally. + /// + /// This is the default behavior. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{self, UnhandledPanic}; + /// + /// # pub fn main() { + /// let rt = runtime::Builder::new_current_thread() + /// .unhandled_panic(UnhandledPanic::Ignore) + /// .build() + /// .unwrap(); + /// + /// let task1 = rt.spawn(async { panic!("boom"); }); + /// let task2 = rt.spawn(async { + /// // This task completes normally + /// "done" + /// }); + /// + /// rt.block_on(async { + /// // The panic on the first task is forwarded to the `JoinHandle` + /// assert!(task1.await.is_err()); + /// + /// // The second task completes normally + /// assert!(task2.await.is_ok()); + /// }) + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + Ignore, + + /// The runtime should immediately shutdown if a spawned task panics. + /// + /// The runtime will immediately shutdown even if the panicked task's + /// [`JoinHandle`] is still available. All further spawned tasks will be + /// immediately dropped and call to [`Runtime::block_on`] will panic. + /// + /// # Examples + /// + /// ```should_panic + /// use tokio::runtime::{self, UnhandledPanic}; + /// + /// # pub fn main() { + /// let rt = runtime::Builder::new_current_thread() + /// .unhandled_panic(UnhandledPanic::ShutdownRuntime) + /// .build() + /// .unwrap(); + /// + /// rt.spawn(async { panic!("boom"); }); + /// rt.spawn(async { + /// // This task never completes. + /// }); + /// + /// rt.block_on(async { + /// // Do some work + /// # loop { tokio::task::yield_now().await; } + /// }) + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + ShutdownRuntime, + } } pub(crate) type ThreadNameFn = std::sync::Arc<dyn Fn() -> String + Send + Sync + 'static>; +#[derive(Clone, Copy)] pub(crate) enum Kind { CurrentThread, - #[cfg(feature = "rt-multi-thread")] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] MultiThread, } @@ -98,28 +199,38 @@ impl Builder { /// /// [`LocalSet`]: crate::task::LocalSet pub fn new_current_thread() -> Builder { - Builder::new(Kind::CurrentThread) + #[cfg(loom)] + const EVENT_INTERVAL: u32 = 4; + // The number `61` is fairly arbitrary. I believe this value was copied from golang. + #[cfg(not(loom))] + const EVENT_INTERVAL: u32 = 61; + + Builder::new(Kind::CurrentThread, 31, EVENT_INTERVAL) } - /// Returns a new builder with the multi thread scheduler selected. - /// - /// Configuration methods can be chained on the return value. - #[cfg(feature = "rt-multi-thread")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] - pub fn new_multi_thread() -> Builder { - Builder::new(Kind::MultiThread) + cfg_not_wasi! { + /// Returns a new builder with the multi thread scheduler selected. + /// + /// Configuration methods can be chained on the return value. + #[cfg(feature = "rt-multi-thread")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] + pub fn new_multi_thread() -> Builder { + // The number `61` is fairly arbitrary. I believe this value was copied from golang. + Builder::new(Kind::MultiThread, 61, 61) + } } /// Returns a new runtime builder initialized with default configuration /// values. /// /// Configuration methods can be chained on the return value. - pub(crate) fn new(kind: Kind) -> Builder { + pub(crate) fn new(kind: Kind, global_queue_interval: u32, event_interval: u32) -> Builder { Builder { kind, // I/O defaults to "off" enable_io: false, + nevents: 1024, // Time defaults to "off" enable_time: false, @@ -127,6 +238,7 @@ impl Builder { // The clock starts not-paused start_paused: false, + // Read from environment variable first in multi-threaded mode. // Default to lazy auto-detection (one thread per CPU core) worker_threads: None, @@ -145,6 +257,18 @@ impl Builder { after_unpark: None, keep_alive: None, + + // Defaults for these values depend on the scheduler kind, so we get them + // as parameters. + global_queue_interval, + event_interval, + + seed_generator: RngSeedGenerator::new(RngSeed::new()), + + #[cfg(tokio_unstable)] + unhandled_panic: UnhandledPanic::Ignore, + + disable_lifo_slot: false, } } @@ -165,7 +289,11 @@ impl Builder { /// .unwrap(); /// ``` pub fn enable_all(&mut self) -> &mut Self { - #[cfg(any(feature = "net", feature = "process", all(unix, feature = "signal")))] + #[cfg(any( + feature = "net", + all(unix, feature = "process"), + all(unix, feature = "signal") + ))] self.enable_io(); #[cfg(feature = "time")] self.enable_time(); @@ -178,15 +306,13 @@ impl Builder { /// This can be any number above 0 though it is advised to keep this value /// on the smaller side. /// + /// This will override the value read from environment variable `TOKIO_WORKER_THREADS`. + /// /// # Default /// /// The default value is the number of cores available to the system. /// - /// # Panic - /// - /// When using the `current_thread` runtime this method will panic, since - /// those variants do not allow setting worker thread counts. - /// + /// When using the `current_thread` runtime this method has no effect. /// /// # Examples /// @@ -219,9 +345,10 @@ impl Builder { /// rt.block_on(async move {}); /// ``` /// - /// # Panic + /// # Panics /// /// This will panic if `val` is not larger than `0`. + #[track_caller] pub fn worker_threads(&mut self, val: usize) -> &mut Self { assert!(val > 0, "Worker threads cannot be set to 0"); self.worker_threads = Some(val); @@ -237,7 +364,7 @@ impl Builder { /// /// The default value is 512. /// - /// # Panic + /// # Panics /// /// This will panic if `val` is not larger than `0`. /// @@ -249,6 +376,7 @@ impl Builder { /// [`spawn_blocking`]: fn@crate::task::spawn_blocking /// [`worker_threads`]: Self::worker_threads /// [`thread_keep_alive`]: Self::thread_keep_alive + #[track_caller] #[cfg_attr(docsrs, doc(alias = "max_threads"))] pub fn max_blocking_threads(&mut self, val: usize) -> &mut Self { assert!(val > 0, "Max blocking threads cannot be set to 0"); @@ -286,7 +414,6 @@ impl Builder { /// ``` /// # use tokio::runtime; /// # use std::sync::atomic::{AtomicUsize, Ordering}; - /// /// # pub fn main() { /// let rt = runtime::Builder::new_multi_thread() /// .thread_name_fn(|| { @@ -338,7 +465,6 @@ impl Builder { /// /// ``` /// # use tokio::runtime; - /// /// # pub fn main() { /// let runtime = runtime::Builder::new_multi_thread() /// .on_thread_start(|| { @@ -364,7 +490,6 @@ impl Builder { /// /// ``` /// # use tokio::runtime; - /// /// # pub fn main() { /// let runtime = runtime::Builder::new_multi_thread() /// .on_thread_stop(|| { @@ -473,7 +598,6 @@ impl Builder { /// /// ``` /// # use tokio::runtime; - /// /// # pub fn main() { /// let runtime = runtime::Builder::new_multi_thread() /// .on_thread_unpark(|| { @@ -513,8 +637,8 @@ impl Builder { /// ``` pub fn build(&mut self) -> io::Result<Runtime> { match &self.kind { - Kind::CurrentThread => self.build_basic_runtime(), - #[cfg(feature = "rt-multi-thread")] + Kind::CurrentThread => self.build_current_thread_runtime(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] Kind::MultiThread => self.build_threaded_runtime(), } } @@ -523,12 +647,13 @@ impl Builder { driver::Cfg { enable_pause_time: match self.kind { Kind::CurrentThread => true, - #[cfg(feature = "rt-multi-thread")] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] Kind::MultiThread => false, }, enable_io: self.enable_io, enable_time: self.enable_time, start_paused: self.start_paused, + nevents: self.nevents, } } @@ -542,7 +667,6 @@ impl Builder { /// ``` /// # use tokio::runtime; /// # use std::time::Duration; - /// /// # pub fn main() { /// let rt = runtime::Builder::new_multi_thread() /// .thread_keep_alive(Duration::from_millis(100)) @@ -554,35 +678,249 @@ impl Builder { self } - fn build_basic_runtime(&mut self) -> io::Result<Runtime> { - use crate::runtime::{BasicScheduler, Kind}; + /// Sets the number of scheduler ticks after which the scheduler will poll the global + /// task queue. + /// + /// A scheduler "tick" roughly corresponds to one `poll` invocation on a task. + /// + /// By default the global queue interval is: + /// + /// * `31` for the current-thread scheduler. + /// * `61` for the multithreaded scheduler. + /// + /// Schedulers have a local queue of already-claimed tasks, and a global queue of incoming + /// tasks. Setting the interval to a smaller value increases the fairness of the scheduler, + /// at the cost of more synchronization overhead. That can be beneficial for prioritizing + /// getting started on new work, especially if tasks frequently yield rather than complete + /// or await on further I/O. Conversely, a higher value prioritizes existing work, and + /// is a good choice when most tasks quickly complete polling. + /// + /// # Examples + /// + /// ``` + /// # use tokio::runtime; + /// # pub fn main() { + /// let rt = runtime::Builder::new_multi_thread() + /// .global_queue_interval(31) + /// .build(); + /// # } + /// ``` + pub fn global_queue_interval(&mut self, val: u32) -> &mut Self { + self.global_queue_interval = val; + self + } - let (driver, resources) = driver::Driver::new(self.get_cfg())?; + /// Sets the number of scheduler ticks after which the scheduler will poll for + /// external events (timers, I/O, and so on). + /// + /// A scheduler "tick" roughly corresponds to one `poll` invocation on a task. + /// + /// By default, the event interval is `61` for all scheduler types. + /// + /// Setting the event interval determines the effective "priority" of delivering + /// these external events (which may wake up additional tasks), compared to + /// executing tasks that are currently ready to run. A smaller value is useful + /// when tasks frequently spend a long time in polling, or frequently yield, + /// which can result in overly long delays picking up I/O events. Conversely, + /// picking up new events requires extra synchronization and syscall overhead, + /// so if tasks generally complete their polling quickly, a higher event interval + /// will minimize that overhead while still keeping the scheduler responsive to + /// events. + /// + /// # Examples + /// + /// ``` + /// # use tokio::runtime; + /// # pub fn main() { + /// let rt = runtime::Builder::new_multi_thread() + /// .event_interval(31) + /// .build(); + /// # } + /// ``` + pub fn event_interval(&mut self, val: u32) -> &mut Self { + self.event_interval = val; + self + } - // And now put a single-threaded scheduler on top of the timer. When - // there are no futures ready to do something, it'll let the timer or - // the reactor to generate some new stimuli for the futures to continue - // in their life. - let scheduler = - BasicScheduler::new(driver, self.before_park.clone(), self.after_unpark.clone()); - let spawner = Spawner::Basic(scheduler.spawner().clone()); + cfg_unstable! { + /// Configure how the runtime responds to an unhandled panic on a + /// spawned task. + /// + /// By default, an unhandled panic (i.e. a panic not caught by + /// [`std::panic::catch_unwind`]) has no impact on the runtime's + /// execution. The panic is error value is forwarded to the task's + /// [`JoinHandle`] and all other spawned tasks continue running. + /// + /// The `unhandled_panic` option enables configuring this behavior. + /// + /// * `UnhandledPanic::Ignore` is the default behavior. Panics on + /// spawned tasks have no impact on the runtime's execution. + /// * `UnhandledPanic::ShutdownRuntime` will force the runtime to + /// shutdown immediately when a spawned task panics even if that + /// task's `JoinHandle` has not been dropped. All other spawned tasks + /// will immediately terminate and further calls to + /// [`Runtime::block_on`] will panic. + /// + /// # Unstable + /// + /// This option is currently unstable and its implementation is + /// incomplete. The API may change or be removed in the future. See + /// tokio-rs/tokio#4516 for more details. + /// + /// # Examples + /// + /// The following demonstrates a runtime configured to shutdown on + /// panic. The first spawned task panics and results in the runtime + /// shutting down. The second spawned task never has a chance to + /// execute. The call to `block_on` will panic due to the runtime being + /// forcibly shutdown. + /// + /// ```should_panic + /// use tokio::runtime::{self, UnhandledPanic}; + /// + /// # pub fn main() { + /// let rt = runtime::Builder::new_current_thread() + /// .unhandled_panic(UnhandledPanic::ShutdownRuntime) + /// .build() + /// .unwrap(); + /// + /// rt.spawn(async { panic!("boom"); }); + /// rt.spawn(async { + /// // This task never completes. + /// }); + /// + /// rt.block_on(async { + /// // Do some work + /// # loop { tokio::task::yield_now().await; } + /// }) + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + pub fn unhandled_panic(&mut self, behavior: UnhandledPanic) -> &mut Self { + self.unhandled_panic = behavior; + self + } + + /// Disables the LIFO task scheduler heuristic. + /// + /// The multi-threaded scheduler includes a heuristic for optimizing + /// message-passing patterns. This heuristic results in the **last** + /// scheduled task being polled first. + /// + /// To implement this heuristic, each worker thread has a slot which + /// holds the task that should be polled next. However, this slot cannot + /// be stolen by other worker threads, which can result in lower total + /// throughput when tasks tend to have longer poll times. + /// + /// This configuration option will disable this heuristic resulting in + /// all scheduled tasks being pushed into the worker-local queue, which + /// is stealable. + /// + /// Consider trying this option when the task "scheduled" time is high + /// but the runtime is underutilized. Use tokio-rs/tokio-metrics to + /// collect this data. + /// + /// # Unstable + /// + /// This configuration option is considered a workaround for the LIFO + /// slot not being stealable. When the slot becomes stealable, we will + /// revisit whether or not this option is necessary. See + /// tokio-rs/tokio#4941. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_multi_thread() + /// .disable_lifo_slot() + /// .build() + /// .unwrap(); + /// ``` + pub fn disable_lifo_slot(&mut self) -> &mut Self { + self.disable_lifo_slot = true; + self + } + + /// Specifies the random number generation seed to use within all + /// threads associated with the runtime being built. + /// + /// This option is intended to make certain parts of the runtime + /// deterministic (e.g. the [`tokio::select!`] macro). In the case of + /// [`tokio::select!`] it will ensure that the order that branches are + /// polled is deterministic. + /// + /// In addition to the code specifying `rng_seed` and interacting with + /// the runtime, the internals of Tokio and the Rust compiler may affect + /// the sequences of random numbers. In order to ensure repeatable + /// results, the version of Tokio, the versions of all other + /// dependencies that interact with Tokio, and the Rust compiler version + /// should also all remain constant. + /// + /// # Examples + /// + /// ``` + /// # use tokio::runtime::{self, RngSeed}; + /// # pub fn main() { + /// let seed = RngSeed::from_bytes(b"place your seed here"); + /// let rt = runtime::Builder::new_current_thread() + /// .rng_seed(seed) + /// .build(); + /// # } + /// ``` + /// + /// [`tokio::select!`]: crate::select + pub fn rng_seed(&mut self, seed: RngSeed) -> &mut Self { + self.seed_generator = RngSeedGenerator::new(seed); + self + } + } + + fn build_current_thread_runtime(&mut self) -> io::Result<Runtime> { + use crate::runtime::scheduler::{self, CurrentThread}; + use crate::runtime::{runtime::Scheduler, Config}; + + let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; // Blocking pool let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads); let blocking_spawner = blocking_pool.spawner().clone(); - Ok(Runtime { - kind: Kind::CurrentThread(scheduler), - handle: Handle { - spawner, - io_handle: resources.io_handle, - time_handle: resources.time_handle, - signal_handle: resources.signal_handle, - clock: resources.clock, - blocking_spawner, + // Generate a rng seed for this runtime. + let seed_generator_1 = self.seed_generator.next_generator(); + let seed_generator_2 = self.seed_generator.next_generator(); + + // And now put a single-threaded scheduler on top of the timer. When + // there are no futures ready to do something, it'll let the timer or + // the reactor to generate some new stimuli for the futures to continue + // in their life. + let (scheduler, handle) = CurrentThread::new( + driver, + driver_handle, + blocking_spawner, + seed_generator_2, + Config { + before_park: self.before_park.clone(), + after_unpark: self.after_unpark.clone(), + global_queue_interval: self.global_queue_interval, + event_interval: self.event_interval, + #[cfg(tokio_unstable)] + unhandled_panic: self.unhandled_panic.clone(), + disable_lifo_slot: self.disable_lifo_slot, + seed_generator: seed_generator_1, }, + ); + + let handle = Handle { + inner: scheduler::Handle::CurrentThread(handle), + }; + + Ok(Runtime::from_parts( + Scheduler::CurrentThread(scheduler), + handle, blocking_pool, - }) + )) } } @@ -607,6 +945,25 @@ cfg_io_driver! { self.enable_io = true; self } + + /// Enables the I/O driver and configures the max number of events to be + /// processed per tick. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_current_thread() + /// .enable_io() + /// .max_io_events_per_tick(1024) + /// .build() + /// .unwrap(); + /// ``` + pub fn max_io_events_per_tick(&mut self, capacity: usize) -> &mut Self { + self.nevents = capacity; + self + } } } @@ -662,39 +1019,47 @@ cfg_rt_multi_thread! { impl Builder { fn build_threaded_runtime(&mut self) -> io::Result<Runtime> { use crate::loom::sys::num_cpus; - use crate::runtime::{Kind, ThreadPool}; - use crate::runtime::park::Parker; + use crate::runtime::{Config, runtime::Scheduler}; + use crate::runtime::scheduler::{self, MultiThread}; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - let (driver, resources) = driver::Driver::new(self.get_cfg())?; - - let (scheduler, launch) = ThreadPool::new(core_threads, Parker::new(driver), self.before_park.clone(), self.after_unpark.clone()); - let spawner = Spawner::ThreadPool(scheduler.spawner().clone()); + let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; // Create the blocking pool - let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads); + let blocking_pool = + blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads); let blocking_spawner = blocking_pool.spawner().clone(); - // Create the runtime handle - let handle = Handle { - spawner, - io_handle: resources.io_handle, - time_handle: resources.time_handle, - signal_handle: resources.signal_handle, - clock: resources.clock, + // Generate a rng seed for this runtime. + let seed_generator_1 = self.seed_generator.next_generator(); + let seed_generator_2 = self.seed_generator.next_generator(); + + let (scheduler, handle, launch) = MultiThread::new( + core_threads, + driver, + driver_handle, blocking_spawner, - }; + seed_generator_2, + Config { + before_park: self.before_park.clone(), + after_unpark: self.after_unpark.clone(), + global_queue_interval: self.global_queue_interval, + event_interval: self.event_interval, + #[cfg(tokio_unstable)] + unhandled_panic: self.unhandled_panic.clone(), + disable_lifo_slot: self.disable_lifo_slot, + seed_generator: seed_generator_1, + }, + ); + + let handle = Handle { inner: scheduler::Handle::MultiThread(handle) }; // Spawn the thread pool workers - let _enter = crate::runtime::context::enter(handle.clone()); + let _enter = handle.enter(); launch.launch(); - Ok(Runtime { - kind: Kind::ThreadPool(scheduler), - handle, - blocking_pool, - }) + Ok(Runtime::from_parts(Scheduler::MultiThread(scheduler), handle, blocking_pool)) } } } diff --git a/src/runtime/config.rs b/src/runtime/config.rs new file mode 100644 index 0000000..39eb1cf --- /dev/null +++ b/src/runtime/config.rs @@ -0,0 +1,34 @@ +#![cfg_attr(any(not(feature = "full"), tokio_wasm), allow(dead_code))] +use crate::runtime::Callback; +use crate::util::RngSeedGenerator; + +pub(crate) struct Config { + /// How many ticks before pulling a task from the global/remote queue? + pub(crate) global_queue_interval: u32, + + /// How many ticks before yielding to the driver for timer and I/O events? + pub(crate) event_interval: u32, + + /// Callback for a worker parking itself + pub(crate) before_park: Option<Callback>, + + /// Callback for a worker unparking itself + pub(crate) after_unpark: Option<Callback>, + + /// The multi-threaded scheduler includes a per-worker LIFO slot used to + /// store the last scheduled task. This can improve certain usage patterns, + /// especially message passing between tasks. However, this LIFO slot is not + /// currently stealable. + /// + /// Eventually, the LIFO slot **will** become stealable, however as a + /// stop-gap, this unstable option lets users disable the LIFO task. + pub(crate) disable_lifo_slot: bool, + + /// Random number generator seed to configure runtimes to act in a + /// deterministic way. + pub(crate) seed_generator: RngSeedGenerator, + + #[cfg(tokio_unstable)] + /// How to respond to unhandled task panics. + pub(crate) unhandled_panic: crate::runtime::UnhandledPanic, +} diff --git a/src/runtime/context.rs b/src/runtime/context.rs index 1f44a53..fef53ca 100644 --- a/src/runtime/context.rs +++ b/src/runtime/context.rs @@ -1,111 +1,420 @@ -//! Thread local runtime context -use crate::runtime::{Handle, TryCurrentError}; +use crate::loom::thread::AccessError; +use crate::runtime::coop; -use std::cell::RefCell; +use std::cell::Cell; -thread_local! { - static CONTEXT: RefCell<Option<Handle>> = RefCell::new(None) +#[cfg(any(feature = "rt", feature = "macros"))] +use crate::util::rand::{FastRand, RngSeed}; + +cfg_rt! { + use crate::runtime::{scheduler, task::Id, Defer}; + + use std::cell::RefCell; + use std::marker::PhantomData; + use std::time::Duration; } -pub(crate) fn try_current() -> Result<Handle, crate::runtime::TryCurrentError> { - match CONTEXT.try_with(|ctx| ctx.borrow().clone()) { - Ok(Some(handle)) => Ok(handle), - Ok(None) => Err(TryCurrentError::new_no_context()), - Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()), - } +struct Context { + /// Uniquely identifies the current thread + #[cfg(feature = "rt")] + thread_id: Cell<Option<ThreadId>>, + + /// Handle to the runtime scheduler running on the current thread. + #[cfg(feature = "rt")] + handle: RefCell<Option<scheduler::Handle>>, + + #[cfg(feature = "rt")] + current_task_id: Cell<Option<Id>>, + + /// Tracks if the current thread is currently driving a runtime. + /// Note, that if this is set to "entered", the current scheduler + /// handle may not reference the runtime currently executing. This + /// is because other runtime handles may be set to current from + /// within a runtime. + #[cfg(feature = "rt")] + runtime: Cell<EnterRuntime>, + + /// Yielded task wakers are stored here and notified after resource drivers + /// are polled. + #[cfg(feature = "rt")] + defer: RefCell<Option<Defer>>, + + #[cfg(any(feature = "rt", feature = "macros"))] + rng: FastRand, + + /// Tracks the amount of "work" a task may still do before yielding back to + /// the sheduler + budget: Cell<coop::Budget>, } -pub(crate) fn current() -> Handle { - match try_current() { - Ok(handle) => handle, - Err(e) => panic!("{}", e), +tokio_thread_local! { + static CONTEXT: Context = { + Context { + #[cfg(feature = "rt")] + thread_id: Cell::new(None), + + /// Tracks the current runtime handle to use when spawning, + /// accessing drivers, etc... + #[cfg(feature = "rt")] + handle: RefCell::new(None), + #[cfg(feature = "rt")] + current_task_id: Cell::new(None), + + /// Tracks if the current thread is currently driving a runtime. + /// Note, that if this is set to "entered", the current scheduler + /// handle may not reference the runtime currently executing. This + /// is because other runtime handles may be set to current from + /// within a runtime. + #[cfg(feature = "rt")] + runtime: Cell::new(EnterRuntime::NotEntered), + + #[cfg(feature = "rt")] + defer: RefCell::new(None), + + #[cfg(any(feature = "rt", feature = "macros"))] + rng: FastRand::new(RngSeed::new()), + + budget: Cell::new(coop::Budget::unconstrained()), + } } } -cfg_io_driver! { - pub(crate) fn io_handle() -> crate::runtime::driver::IoHandle { - match CONTEXT.try_with(|ctx| { - let ctx = ctx.borrow(); - ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).io_handle.clone() - }) { - Ok(io_handle) => io_handle, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), +#[cfg(feature = "macros")] +pub(crate) fn thread_rng_n(n: u32) -> u32 { + CONTEXT.with(|ctx| ctx.rng.fastrand_n(n)) +} + +pub(super) fn budget<R>(f: impl FnOnce(&Cell<coop::Budget>) -> R) -> Result<R, AccessError> { + CONTEXT.try_with(|ctx| f(&ctx.budget)) +} + +cfg_rt! { + use crate::runtime::{ThreadId, TryCurrentError}; + + use std::fmt; + + pub(crate) fn thread_id() -> Result<ThreadId, AccessError> { + CONTEXT.try_with(|ctx| { + match ctx.thread_id.get() { + Some(id) => id, + None => { + let id = ThreadId::next(); + ctx.thread_id.set(Some(id)); + id + } + } + }) + } + + #[derive(Debug, Clone, Copy)] + #[must_use] + pub(crate) enum EnterRuntime { + /// Currently in a runtime context. + #[cfg_attr(not(feature = "rt"), allow(dead_code))] + Entered { allow_block_in_place: bool }, + + /// Not in a runtime context **or** a blocking region. + NotEntered, + } + + #[derive(Debug)] + #[must_use] + pub(crate) struct SetCurrentGuard { + old_handle: Option<scheduler::Handle>, + old_seed: RngSeed, + } + + /// Guard tracking that a caller has entered a runtime context. + #[must_use] + pub(crate) struct EnterRuntimeGuard { + /// Tracks that the current thread has entered a blocking function call. + pub(crate) blocking: BlockingRegionGuard, + + #[allow(dead_code)] // Only tracking the guard. + pub(crate) handle: SetCurrentGuard, + + /// If true, then this is the root runtime guard. It is possible to nest + /// runtime guards by using `block_in_place` between the calls. We need + /// to track the root guard as this is the guard responsible for freeing + /// the deferred task queue. + is_root: bool, + } + + /// Guard tracking that a caller has entered a blocking region. + #[must_use] + pub(crate) struct BlockingRegionGuard { + _p: PhantomData<RefCell<()>>, + } + + pub(crate) struct DisallowBlockInPlaceGuard(bool); + + pub(crate) fn set_current_task_id(id: Option<Id>) -> Option<Id> { + CONTEXT.try_with(|ctx| ctx.current_task_id.replace(id)).unwrap_or(None) + } + + pub(crate) fn current_task_id() -> Option<Id> { + CONTEXT.try_with(|ctx| ctx.current_task_id.get()).unwrap_or(None) + } + + pub(crate) fn try_current() -> Result<scheduler::Handle, TryCurrentError> { + match CONTEXT.try_with(|ctx| ctx.handle.borrow().clone()) { + Ok(Some(handle)) => Ok(handle), + Ok(None) => Err(TryCurrentError::new_no_context()), + Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()), } } -} -cfg_signal_internal! { - #[cfg(unix)] - pub(crate) fn signal_handle() -> crate::runtime::driver::SignalHandle { - match CONTEXT.try_with(|ctx| { - let ctx = ctx.borrow(); - ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).signal_handle.clone() - }) { - Ok(signal_handle) => signal_handle, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + /// Sets this [`Handle`] as the current active [`Handle`]. + /// + /// [`Handle`]: crate::runtime::scheduler::Handle + pub(crate) fn try_set_current(handle: &scheduler::Handle) -> Option<SetCurrentGuard> { + CONTEXT.try_with(|ctx| ctx.set_current(handle)).ok() + } + + + /// Marks the current thread as being within the dynamic extent of an + /// executor. + #[track_caller] + pub(crate) fn enter_runtime(handle: &scheduler::Handle, allow_block_in_place: bool) -> EnterRuntimeGuard { + if let Some(enter) = try_enter_runtime(handle, allow_block_in_place) { + return enter; } + + panic!( + "Cannot start a runtime from within a runtime. This happens \ + because a function (like `block_on`) attempted to block the \ + current thread while the thread is being used to drive \ + asynchronous tasks." + ); } -} -cfg_time! { - pub(crate) fn time_handle() -> crate::runtime::driver::TimeHandle { - match CONTEXT.try_with(|ctx| { - let ctx = ctx.borrow(); - ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).time_handle.clone() - }) { - Ok(time_handle) => time_handle, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + /// Tries to enter a runtime context, returns `None` if already in a runtime + /// context. + fn try_enter_runtime(handle: &scheduler::Handle, allow_block_in_place: bool) -> Option<EnterRuntimeGuard> { + CONTEXT.with(|c| { + if c.runtime.get().is_entered() { + None + } else { + // Set the entered flag + c.runtime.set(EnterRuntime::Entered { allow_block_in_place }); + + // Initialize queue to track yielded tasks + let mut defer = c.defer.borrow_mut(); + + let is_root = if defer.is_none() { + *defer = Some(Defer::new()); + true + } else { + false + }; + + Some(EnterRuntimeGuard { + blocking: BlockingRegionGuard::new(), + handle: c.set_current(handle), + is_root, + }) + } + }) + } + + pub(crate) fn try_enter_blocking_region() -> Option<BlockingRegionGuard> { + CONTEXT.try_with(|c| { + if c.runtime.get().is_entered() { + None + } else { + Some(BlockingRegionGuard::new()) + } + // If accessing the thread-local fails, the thread is terminating + // and thread-locals are being destroyed. Because we don't know if + // we are currently in a runtime or not, we default to being + // permissive. + }).unwrap_or_else(|_| Some(BlockingRegionGuard::new())) + } + + /// Disallows blocking in the current runtime context until the guard is dropped. + pub(crate) fn disallow_block_in_place() -> DisallowBlockInPlaceGuard { + let reset = CONTEXT.with(|c| { + if let EnterRuntime::Entered { + allow_block_in_place: true, + } = c.runtime.get() + { + c.runtime.set(EnterRuntime::Entered { + allow_block_in_place: false, + }); + true + } else { + false + } + }); + + DisallowBlockInPlaceGuard(reset) + } + + pub(crate) fn with_defer<R>(f: impl FnOnce(&mut Defer) -> R) -> Option<R> { + CONTEXT.with(|c| { + let mut defer = c.defer.borrow_mut(); + defer.as_mut().map(f) + }) + } + + impl Context { + fn set_current(&self, handle: &scheduler::Handle) -> SetCurrentGuard { + let rng_seed = handle.seed_generator().next_seed(); + + let old_handle = self.handle.borrow_mut().replace(handle.clone()); + let old_seed = self.rng.replace_seed(rng_seed); + + SetCurrentGuard { + old_handle, + old_seed, + } } } - cfg_test_util! { - pub(crate) fn clock() -> Option<crate::runtime::driver::Clock> { - match CONTEXT.try_with(|ctx| (*ctx.borrow()).as_ref().map(|ctx| ctx.clock.clone())) { - Ok(clock) => clock, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + impl Drop for SetCurrentGuard { + fn drop(&mut self) { + CONTEXT.with(|ctx| { + *ctx.handle.borrow_mut() = self.old_handle.take(); + ctx.rng.replace_seed(self.old_seed.clone()); + }); + } + } + + impl fmt::Debug for EnterRuntimeGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Enter").finish() + } + } + + impl Drop for EnterRuntimeGuard { + fn drop(&mut self) { + CONTEXT.with(|c| { + assert!(c.runtime.get().is_entered()); + c.runtime.set(EnterRuntime::NotEntered); + + if self.is_root { + *c.defer.borrow_mut() = None; + } + }); + } + } + + impl BlockingRegionGuard { + fn new() -> BlockingRegionGuard { + BlockingRegionGuard { _p: PhantomData } + } + /// Blocks the thread on the specified future, returning the value with + /// which that future completes. + pub(crate) fn block_on<F>(&mut self, f: F) -> Result<F::Output, AccessError> + where + F: std::future::Future, + { + use crate::runtime::park::CachedParkThread; + + let mut park = CachedParkThread::new(); + park.block_on(f) + } + + /// Blocks the thread on the specified future for **at most** `timeout` + /// + /// If the future completes before `timeout`, the result is returned. If + /// `timeout` elapses, then `Err` is returned. + pub(crate) fn block_on_timeout<F>(&mut self, f: F, timeout: Duration) -> Result<F::Output, ()> + where + F: std::future::Future, + { + use crate::runtime::park::CachedParkThread; + use std::task::Context; + use std::task::Poll::Ready; + use std::time::Instant; + + let mut park = CachedParkThread::new(); + let waker = park.waker().map_err(|_| ())?; + let mut cx = Context::from_waker(&waker); + + pin!(f); + let when = Instant::now() + timeout; + + loop { + if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) { + return Ok(v); + } + + let now = Instant::now(); + + if now >= when { + return Err(()); + } + + // Wake any yielded tasks before parking in order to avoid + // blocking. + with_defer(|defer| defer.wake()); + + park.park_timeout(when - now); } } } -} -cfg_rt! { - pub(crate) fn spawn_handle() -> Option<crate::runtime::Spawner> { - match CONTEXT.try_with(|ctx| (*ctx.borrow()).as_ref().map(|ctx| ctx.spawner.clone())) { - Ok(spawner) => spawner, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + impl Drop for DisallowBlockInPlaceGuard { + fn drop(&mut self) { + if self.0 { + // XXX: Do we want some kind of assertion here, or is "best effort" okay? + CONTEXT.with(|c| { + if let EnterRuntime::Entered { + allow_block_in_place: false, + } = c.runtime.get() + { + c.runtime.set(EnterRuntime::Entered { + allow_block_in_place: true, + }); + } + }) + } } } -} -/// Sets this [`Handle`] as the current active [`Handle`]. -/// -/// [`Handle`]: Handle -pub(crate) fn enter(new: Handle) -> EnterGuard { - match try_enter(new) { - Some(guard) => guard, - None => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + impl EnterRuntime { + pub(crate) fn is_entered(self) -> bool { + matches!(self, EnterRuntime::Entered { .. }) + } } } -/// Sets this [`Handle`] as the current active [`Handle`]. -/// -/// [`Handle`]: Handle -pub(crate) fn try_enter(new: Handle) -> Option<EnterGuard> { - CONTEXT - .try_with(|ctx| { - let old = ctx.borrow_mut().replace(new); - EnterGuard(old) - }) - .ok() -} +// Forces the current "entered" state to be cleared while the closure +// is executed. +// +// # Warning +// +// This is hidden for a reason. Do not use without fully understanding +// executors. Misusing can easily cause your program to deadlock. +cfg_rt_multi_thread! { + /// Returns true if in a runtime context. + pub(crate) fn current_enter_context() -> EnterRuntime { + CONTEXT.with(|c| c.runtime.get()) + } + + pub(crate) fn exit_runtime<F: FnOnce() -> R, R>(f: F) -> R { + // Reset in case the closure panics + struct Reset(EnterRuntime); -#[derive(Debug)] -pub(crate) struct EnterGuard(Option<Handle>); + impl Drop for Reset { + fn drop(&mut self) { + CONTEXT.with(|c| { + assert!(!c.runtime.get().is_entered(), "closure claimed permanent executor"); + c.runtime.set(self.0); + }); + } + } -impl Drop for EnterGuard { - fn drop(&mut self) { - CONTEXT.with(|ctx| { - *ctx.borrow_mut() = self.0.take(); + let was = CONTEXT.with(|c| { + let e = c.runtime.get(); + assert!(e.is_entered(), "asked to exit when not entered"); + c.runtime.set(EnterRuntime::NotEntered); + e }); + + let _reset = Reset(was); + // dropping _reset after f() will reset ENTERED + f() } } diff --git a/src/coop.rs b/src/runtime/coop.rs index 256e962..0ba137a 100644 --- a/src/coop.rs +++ b/src/runtime/coop.rs @@ -29,11 +29,7 @@ // other futures. By doing this, you avoid double-counting each iteration of // the outer future against the cooperating budget. -use std::cell::Cell; - -thread_local! { - static CURRENT: Cell<Budget> = Cell::new(Budget::unconstrained()); -} +use crate::runtime::context; /// Opaque type tracking the amount of "work" a task may still do before /// yielding back to the scheduler. @@ -56,16 +52,12 @@ impl Budget { } /// Returns an unconstrained budget. Operations will not be limited. - const fn unconstrained() -> Budget { + pub(super) const fn unconstrained() -> Budget { Budget(None) } -} -cfg_rt_multi_thread! { - impl Budget { - fn has_remaining(self) -> bool { - self.0.map(|budget| budget > 0).unwrap_or(true) - } + fn has_remaining(self) -> bool { + self.0.map(|budget| budget > 0).unwrap_or(true) } } @@ -85,37 +77,42 @@ pub(crate) fn with_unconstrained<R>(f: impl FnOnce() -> R) -> R { #[inline(always)] fn with_budget<R>(budget: Budget, f: impl FnOnce() -> R) -> R { - struct ResetGuard<'a> { - cell: &'a Cell<Budget>, + struct ResetGuard { prev: Budget, } - impl<'a> Drop for ResetGuard<'a> { + impl Drop for ResetGuard { fn drop(&mut self) { - self.cell.set(self.prev); + let _ = context::budget(|cell| { + cell.set(self.prev); + }); } } - CURRENT.with(move |cell| { + #[allow(unused_variables)] + let maybe_guard = context::budget(|cell| { let prev = cell.get(); - cell.set(budget); - let _guard = ResetGuard { cell, prev }; + ResetGuard { prev } + }); + + // The function is called regardless even if the budget is not successfully + // set due to the thread-local being destroyed. + f() +} - f() - }) +#[inline(always)] +pub(crate) fn has_budget_remaining() -> bool { + // If the current budget cannot be accessed due to the thread-local being + // shutdown, then we assume there is budget remaining. + context::budget(|cell| cell.get().has_remaining()).unwrap_or(true) } cfg_rt_multi_thread! { /// Sets the current task's budget. pub(crate) fn set(budget: Budget) { - CURRENT.with(|cell| cell.set(budget)) - } - - #[inline(always)] - pub(crate) fn has_budget_remaining() -> bool { - CURRENT.with(|cell| cell.get().has_remaining()) + let _ = context::budget(|cell| cell.set(budget)); } } @@ -124,15 +121,16 @@ cfg_rt! { /// /// Returns the remaining budget pub(crate) fn stop() -> Budget { - CURRENT.with(|cell| { + context::budget(|cell| { let prev = cell.get(); cell.set(Budget::unconstrained()); prev - }) + }).unwrap_or(Budget::unconstrained()) } } cfg_coop! { + use std::cell::Cell; use std::task::{Context, Poll}; #[must_use] @@ -150,7 +148,7 @@ cfg_coop! { // They are both represented as the remembered budget being unconstrained. let budget = self.0.get(); if !budget.is_unconstrained() { - CURRENT.with(|cell| { + let _ = context::budget(|cell| { cell.set(budget); }); } @@ -166,12 +164,12 @@ cfg_coop! { /// that the budget empties appropriately. /// /// Note that `RestoreOnPending` restores the budget **as it was before `poll_proceed`**. - /// Therefore, if the budget is _further_ adjusted between when `poll_proceed` returns and + /// Therefore, if the budget is _fCURRENT.withurther_ adjusted between when `poll_proceed` returns and /// `RestRestoreOnPending` is dropped, those adjustments are erased unless the caller indicates /// that progress was made. #[inline] pub(crate) fn poll_proceed(cx: &mut Context<'_>) -> Poll<RestoreOnPending> { - CURRENT.with(|cell| { + context::budget(|cell| { let mut budget = cell.get(); if budget.decrement() { @@ -182,7 +180,7 @@ cfg_coop! { cx.waker().wake_by_ref(); Poll::Pending } - }) + }).unwrap_or(Poll::Ready(RestoreOnPending(Cell::new(Budget::unconstrained())))) } impl Budget { @@ -211,12 +209,15 @@ cfg_coop! { mod test { use super::*; + #[cfg(tokio_wasm_not_wasi)] + use wasm_bindgen_test::wasm_bindgen_test as test; + fn get() -> Budget { - CURRENT.with(|cell| cell.get()) + context::budget(|cell| cell.get()).unwrap_or(Budget::unconstrained()) } #[test] - fn bugeting() { + fn budgeting() { use futures::future::poll_fn; use tokio_test::*; diff --git a/src/runtime/defer.rs b/src/runtime/defer.rs new file mode 100644 index 0000000..4078512 --- /dev/null +++ b/src/runtime/defer.rs @@ -0,0 +1,27 @@ +use std::task::Waker; + +pub(crate) struct Defer { + deferred: Vec<Waker>, +} + +impl Defer { + pub(crate) fn new() -> Defer { + Defer { + deferred: Default::default(), + } + } + + pub(crate) fn defer(&mut self, waker: Waker) { + self.deferred.push(waker); + } + + pub(crate) fn is_empty(&self) -> bool { + self.deferred.is_empty() + } + + pub(crate) fn wake(&mut self) { + for waker in self.deferred.drain(..) { + waker.wake(); + } + } +} diff --git a/src/runtime/driver.rs b/src/runtime/driver.rs index 7e45977..4fb6b87 100644 --- a/src/runtime/driver.rs +++ b/src/runtime/driver.rs @@ -1,63 +1,236 @@ //! Abstracts out the entire chain of runtime sub-drivers into common types. -use crate::park::thread::ParkThread; -use crate::park::Park; + +// Eventually, this file will see significant refactoring / cleanup. For now, we +// don't need to worry much about dead code with certain feature permutations. +#![cfg_attr(not(feature = "full"), allow(dead_code))] + +use crate::runtime::park::{ParkThread, UnparkThread}; use std::io; use std::time::Duration; +#[derive(Debug)] +pub(crate) struct Driver { + inner: TimeDriver, +} + +#[derive(Debug)] +pub(crate) struct Handle { + /// IO driver handle + pub(crate) io: IoHandle, + + /// Signal driver handle + #[cfg_attr(any(not(unix), loom), allow(dead_code))] + pub(crate) signal: SignalHandle, + + /// Time driver handle + pub(crate) time: TimeHandle, + + /// Source of `Instant::now()` + #[cfg_attr(not(all(feature = "time", feature = "test-util")), allow(dead_code))] + pub(crate) clock: Clock, +} + +pub(crate) struct Cfg { + pub(crate) enable_io: bool, + pub(crate) enable_time: bool, + pub(crate) enable_pause_time: bool, + pub(crate) start_paused: bool, + pub(crate) nevents: usize, +} + +impl Driver { + pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Handle)> { + let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io, cfg.nevents)?; + + let clock = create_clock(cfg.enable_pause_time, cfg.start_paused); + + let (time_driver, time_handle) = + create_time_driver(cfg.enable_time, io_stack, clock.clone()); + + Ok(( + Self { inner: time_driver }, + Handle { + io: io_handle, + signal: signal_handle, + time: time_handle, + clock, + }, + )) + } + + pub(crate) fn park(&mut self, handle: &Handle) { + self.inner.park(handle) + } + + pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) { + self.inner.park_timeout(handle, duration) + } + + pub(crate) fn shutdown(&mut self, handle: &Handle) { + self.inner.shutdown(handle) + } +} + +impl Handle { + pub(crate) fn unpark(&self) { + #[cfg(feature = "time")] + if let Some(handle) = &self.time { + handle.unpark(); + } + + self.io.unpark(); + } + + cfg_io_driver! { + #[track_caller] + pub(crate) fn io(&self) -> &crate::runtime::io::Handle { + self.io + .as_ref() + .expect("A Tokio 1.x context was found, but IO is disabled. Call `enable_io` on the runtime builder to enable IO.") + } + } + + cfg_signal_internal_and_unix! { + #[track_caller] + pub(crate) fn signal(&self) -> &crate::runtime::signal::Handle { + self.signal + .as_ref() + .expect("there is no signal driver running, must be called from the context of Tokio runtime") + } + } + + cfg_time! { + /// Returns a reference to the time driver handle. + /// + /// Panics if no time driver is present. + #[track_caller] + pub(crate) fn time(&self) -> &crate::runtime::time::Handle { + self.time + .as_ref() + .expect("A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers.") + } + + cfg_test_util! { + pub(crate) fn clock(&self) -> &Clock { + &self.clock + } + } + } +} + // ===== io driver ===== cfg_io_driver! { - type IoDriver = crate::io::driver::Driver; - type IoStack = crate::park::either::Either<ProcessDriver, ParkThread>; - pub(crate) type IoHandle = Option<crate::io::driver::Handle>; + pub(crate) type IoDriver = crate::runtime::io::Driver; - fn create_io_stack(enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> { - use crate::park::either::Either; + #[derive(Debug)] + pub(crate) enum IoStack { + Enabled(ProcessDriver), + Disabled(ParkThread), + } + + #[derive(Debug)] + pub(crate) enum IoHandle { + Enabled(crate::runtime::io::Handle), + Disabled(UnparkThread), + } + fn create_io_stack(enabled: bool, nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> { #[cfg(loom)] assert!(!enabled); let ret = if enabled { - let io_driver = crate::io::driver::Driver::new()?; - let io_handle = io_driver.handle(); + let (io_driver, io_handle) = crate::runtime::io::Driver::new(nevents)?; - let (signal_driver, signal_handle) = create_signal_driver(io_driver)?; + let (signal_driver, signal_handle) = create_signal_driver(io_driver, &io_handle)?; let process_driver = create_process_driver(signal_driver); - (Either::A(process_driver), Some(io_handle), signal_handle) + (IoStack::Enabled(process_driver), IoHandle::Enabled(io_handle), signal_handle) } else { - (Either::B(ParkThread::new()), Default::default(), Default::default()) + let park_thread = ParkThread::new(); + let unpark_thread = park_thread.unpark(); + (IoStack::Disabled(park_thread), IoHandle::Disabled(unpark_thread), Default::default()) }; Ok(ret) } + + impl IoStack { + pub(crate) fn park(&mut self, handle: &Handle) { + match self { + IoStack::Enabled(v) => v.park(handle), + IoStack::Disabled(v) => v.park(), + } + } + + pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) { + match self { + IoStack::Enabled(v) => v.park_timeout(handle, duration), + IoStack::Disabled(v) => v.park_timeout(duration), + } + } + + pub(crate) fn shutdown(&mut self, handle: &Handle) { + match self { + IoStack::Enabled(v) => v.shutdown(handle), + IoStack::Disabled(v) => v.shutdown(), + } + } + } + + impl IoHandle { + pub(crate) fn unpark(&self) { + match self { + IoHandle::Enabled(handle) => handle.unpark(), + IoHandle::Disabled(handle) => handle.unpark(), + } + } + + pub(crate) fn as_ref(&self) -> Option<&crate::runtime::io::Handle> { + match self { + IoHandle::Enabled(v) => Some(v), + IoHandle::Disabled(..) => None, + } + } + } } cfg_not_io_driver! { - pub(crate) type IoHandle = (); - type IoStack = ParkThread; + pub(crate) type IoHandle = UnparkThread; - fn create_io_stack(_enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> { - Ok((ParkThread::new(), Default::default(), Default::default())) + #[derive(Debug)] + pub(crate) struct IoStack(ParkThread); + + fn create_io_stack(_enabled: bool, _nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> { + let park_thread = ParkThread::new(); + let unpark_thread = park_thread.unpark(); + Ok((IoStack(park_thread), unpark_thread, Default::default())) } -} -// ===== signal driver ===== + impl IoStack { + pub(crate) fn park(&mut self, _handle: &Handle) { + self.0.park(); + } + + pub(crate) fn park_timeout(&mut self, _handle: &Handle, duration: Duration) { + self.0.park_timeout(duration); + } -macro_rules! cfg_signal_internal_and_unix { - ($($item:item)*) => { - #[cfg(unix)] - cfg_signal_internal! { $($item)* } + pub(crate) fn shutdown(&mut self, _handle: &Handle) { + self.0.shutdown(); + } } } +// ===== signal driver ===== + cfg_signal_internal_and_unix! { - type SignalDriver = crate::signal::unix::driver::Driver; - pub(crate) type SignalHandle = Option<crate::signal::unix::driver::Handle>; + type SignalDriver = crate::runtime::signal::Driver; + pub(crate) type SignalHandle = Option<crate::runtime::signal::Handle>; - fn create_signal_driver(io_driver: IoDriver) -> io::Result<(SignalDriver, SignalHandle)> { - let driver = crate::signal::unix::driver::Driver::new(io_driver)?; + fn create_signal_driver(io_driver: IoDriver, io_handle: &crate::runtime::io::Handle) -> io::Result<(SignalDriver, SignalHandle)> { + let driver = crate::runtime::signal::Driver::new(io_driver, io_handle)?; let handle = driver.handle(); Ok((driver, Some(handle))) } @@ -69,7 +242,7 @@ cfg_not_signal_internal! { cfg_io_driver! { type SignalDriver = IoDriver; - fn create_signal_driver(io_driver: IoDriver) -> io::Result<(SignalDriver, SignalHandle)> { + fn create_signal_driver(io_driver: IoDriver, _io_handle: &crate::runtime::io::Handle) -> io::Result<(SignalDriver, SignalHandle)> { Ok((io_driver, ())) } } @@ -78,10 +251,10 @@ cfg_not_signal_internal! { // ===== process driver ===== cfg_process_driver! { - type ProcessDriver = crate::process::unix::driver::Driver; + type ProcessDriver = crate::runtime::process::Driver; fn create_process_driver(signal_driver: SignalDriver) -> ProcessDriver { - crate::process::unix::driver::Driver::new(signal_driver) + ProcessDriver::new(signal_driver) } } @@ -98,10 +271,16 @@ cfg_not_process_driver! { // ===== time driver ===== cfg_time! { - type TimeDriver = crate::park::either::Either<crate::time::driver::Driver<IoStack>, IoStack>; + #[derive(Debug)] + pub(crate) enum TimeDriver { + Enabled { + driver: crate::runtime::time::Driver, + }, + Disabled(IoStack), + } pub(crate) type Clock = crate::time::Clock; - pub(crate) type TimeHandle = Option<crate::time::driver::Handle>; + pub(crate) type TimeHandle = Option<crate::runtime::time::Handle>; fn create_clock(enable_pausing: bool, start_paused: bool) -> Clock { crate::time::Clock::new(enable_pausing, start_paused) @@ -112,15 +291,35 @@ cfg_time! { io_stack: IoStack, clock: Clock, ) -> (TimeDriver, TimeHandle) { - use crate::park::either::Either; - if enable { - let driver = crate::time::driver::Driver::new(io_stack, clock); - let handle = driver.handle(); + let (driver, handle) = crate::runtime::time::Driver::new(io_stack, clock); - (Either::A(driver), Some(handle)) + (TimeDriver::Enabled { driver }, Some(handle)) } else { - (Either::B(io_stack), None) + (TimeDriver::Disabled(io_stack), None) + } + } + + impl TimeDriver { + pub(crate) fn park(&mut self, handle: &Handle) { + match self { + TimeDriver::Enabled { driver, .. } => driver.park(handle), + TimeDriver::Disabled(v) => v.park(handle), + } + } + + pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) { + match self { + TimeDriver::Enabled { driver } => driver.park_timeout(handle, duration), + TimeDriver::Disabled(v) => v.park_timeout(handle, duration), + } + } + + pub(crate) fn shutdown(&mut self, handle: &Handle) { + match self { + TimeDriver::Enabled { driver } => driver.shutdown(handle), + TimeDriver::Disabled(v) => v.shutdown(handle), + } } } } @@ -143,66 +342,3 @@ cfg_not_time! { (io_stack, ()) } } - -// ===== runtime driver ===== - -#[derive(Debug)] -pub(crate) struct Driver { - inner: TimeDriver, -} - -pub(crate) struct Resources { - pub(crate) io_handle: IoHandle, - pub(crate) signal_handle: SignalHandle, - pub(crate) time_handle: TimeHandle, - pub(crate) clock: Clock, -} - -pub(crate) struct Cfg { - pub(crate) enable_io: bool, - pub(crate) enable_time: bool, - pub(crate) enable_pause_time: bool, - pub(crate) start_paused: bool, -} - -impl Driver { - pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Resources)> { - let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io)?; - - let clock = create_clock(cfg.enable_pause_time, cfg.start_paused); - - let (time_driver, time_handle) = - create_time_driver(cfg.enable_time, io_stack, clock.clone()); - - Ok(( - Self { inner: time_driver }, - Resources { - io_handle, - signal_handle, - time_handle, - clock, - }, - )) - } -} - -impl Park for Driver { - type Unpark = <TimeDriver as Park>::Unpark; - type Error = <TimeDriver as Park>::Error; - - fn unpark(&self) -> Self::Unpark { - self.inner.unpark() - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.inner.park() - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.inner.park_timeout(duration) - } - - fn shutdown(&mut self) { - self.inner.shutdown() - } -} diff --git a/src/runtime/enter.rs b/src/runtime/enter.rs deleted file mode 100644 index 3f14cb5..0000000 --- a/src/runtime/enter.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::cell::{Cell, RefCell}; -use std::fmt; -use std::marker::PhantomData; - -#[derive(Debug, Clone, Copy)] -pub(crate) enum EnterContext { - #[cfg_attr(not(feature = "rt"), allow(dead_code))] - Entered { - allow_blocking: bool, - }, - NotEntered, -} - -impl EnterContext { - pub(crate) fn is_entered(self) -> bool { - matches!(self, EnterContext::Entered { .. }) - } -} - -thread_local!(static ENTERED: Cell<EnterContext> = Cell::new(EnterContext::NotEntered)); - -/// Represents an executor context. -pub(crate) struct Enter { - _p: PhantomData<RefCell<()>>, -} - -cfg_rt! { - use crate::park::thread::ParkError; - - use std::time::Duration; - - /// Marks the current thread as being within the dynamic extent of an - /// executor. - pub(crate) fn enter(allow_blocking: bool) -> Enter { - if let Some(enter) = try_enter(allow_blocking) { - return enter; - } - - panic!( - "Cannot start a runtime from within a runtime. This happens \ - because a function (like `block_on`) attempted to block the \ - current thread while the thread is being used to drive \ - asynchronous tasks." - ); - } - - /// Tries to enter a runtime context, returns `None` if already in a runtime - /// context. - pub(crate) fn try_enter(allow_blocking: bool) -> Option<Enter> { - ENTERED.with(|c| { - if c.get().is_entered() { - None - } else { - c.set(EnterContext::Entered { allow_blocking }); - Some(Enter { _p: PhantomData }) - } - }) - } -} - -// Forces the current "entered" state to be cleared while the closure -// is executed. -// -// # Warning -// -// This is hidden for a reason. Do not use without fully understanding -// executors. Misusing can easily cause your program to deadlock. -cfg_rt_multi_thread! { - pub(crate) fn exit<F: FnOnce() -> R, R>(f: F) -> R { - // Reset in case the closure panics - struct Reset(EnterContext); - impl Drop for Reset { - fn drop(&mut self) { - ENTERED.with(|c| { - assert!(!c.get().is_entered(), "closure claimed permanent executor"); - c.set(self.0); - }); - } - } - - let was = ENTERED.with(|c| { - let e = c.get(); - assert!(e.is_entered(), "asked to exit when not entered"); - c.set(EnterContext::NotEntered); - e - }); - - let _reset = Reset(was); - // dropping _reset after f() will reset ENTERED - f() - } -} - -cfg_rt! { - /// Disallows blocking in the current runtime context until the guard is dropped. - pub(crate) fn disallow_blocking() -> DisallowBlockingGuard { - let reset = ENTERED.with(|c| { - if let EnterContext::Entered { - allow_blocking: true, - } = c.get() - { - c.set(EnterContext::Entered { - allow_blocking: false, - }); - true - } else { - false - } - }); - DisallowBlockingGuard(reset) - } - - pub(crate) struct DisallowBlockingGuard(bool); - impl Drop for DisallowBlockingGuard { - fn drop(&mut self) { - if self.0 { - // XXX: Do we want some kind of assertion here, or is "best effort" okay? - ENTERED.with(|c| { - if let EnterContext::Entered { - allow_blocking: false, - } = c.get() - { - c.set(EnterContext::Entered { - allow_blocking: true, - }); - } - }) - } - } - } -} - -cfg_rt_multi_thread! { - /// Returns true if in a runtime context. - pub(crate) fn context() -> EnterContext { - ENTERED.with(|c| c.get()) - } -} - -cfg_rt! { - impl Enter { - /// Blocks the thread on the specified future, returning the value with - /// which that future completes. - pub(crate) fn block_on<F>(&mut self, f: F) -> Result<F::Output, ParkError> - where - F: std::future::Future, - { - use crate::park::thread::CachedParkThread; - - let mut park = CachedParkThread::new(); - park.block_on(f) - } - - /// Blocks the thread on the specified future for **at most** `timeout` - /// - /// If the future completes before `timeout`, the result is returned. If - /// `timeout` elapses, then `Err` is returned. - pub(crate) fn block_on_timeout<F>(&mut self, f: F, timeout: Duration) -> Result<F::Output, ParkError> - where - F: std::future::Future, - { - use crate::park::Park; - use crate::park::thread::CachedParkThread; - use std::task::Context; - use std::task::Poll::Ready; - use std::time::Instant; - - let mut park = CachedParkThread::new(); - let waker = park.get_unpark()?.into_waker(); - let mut cx = Context::from_waker(&waker); - - pin!(f); - let when = Instant::now() + timeout; - - loop { - if let Ready(v) = crate::coop::budget(|| f.as_mut().poll(&mut cx)) { - return Ok(v); - } - - let now = Instant::now(); - - if now >= when { - return Err(()); - } - - park.park_timeout(when - now)?; - } - } - } -} - -impl fmt::Debug for Enter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Enter").finish() - } -} - -impl Drop for Enter { - fn drop(&mut self) { - ENTERED.with(|c| { - assert!(c.get().is_entered()); - c.set(EnterContext::NotEntered); - }); - } -} diff --git a/src/runtime/handle.rs b/src/runtime/handle.rs index cd1cb76..c5dc65f 100644 --- a/src/runtime/handle.rs +++ b/src/runtime/handle.rs @@ -1,11 +1,4 @@ -use crate::runtime::blocking::{BlockingTask, NoopSchedule}; -use crate::runtime::task::{self, JoinHandle}; -use crate::runtime::{blocking, context, driver, Spawner}; -use crate::util::error::{CONTEXT_MISSING_ERROR, THREAD_LOCAL_DESTROYED_ERROR}; - -use std::future::Future; -use std::marker::PhantomData; -use std::{error, fmt}; +use crate::runtime::{context, scheduler, RuntimeFlavor}; /// Handle to the runtime. /// @@ -14,34 +7,18 @@ use std::{error, fmt}; /// /// [`Runtime::handle`]: crate::runtime::Runtime::handle() #[derive(Debug, Clone)] +// When the `rt` feature is *not* enabled, this type is still defined, but not +// included in the public API. pub struct Handle { - pub(super) spawner: Spawner, - - /// Handles to the I/O drivers - #[cfg_attr( - not(any(feature = "net", feature = "process", all(unix, feature = "signal"))), - allow(dead_code) - )] - pub(super) io_handle: driver::IoHandle, - - /// Handles to the signal drivers - #[cfg_attr( - not(any(feature = "signal", all(unix, feature = "process"))), - allow(dead_code) - )] - pub(super) signal_handle: driver::SignalHandle, - - /// Handles to the time drivers - #[cfg_attr(not(feature = "time"), allow(dead_code))] - pub(super) time_handle: driver::TimeHandle, + pub(crate) inner: scheduler::Handle, +} - /// Source of `Instant::now()` - #[cfg_attr(not(all(feature = "time", feature = "test-util")), allow(dead_code))] - pub(super) clock: driver::Clock, +use crate::runtime::task::JoinHandle; +use crate::util::error::{CONTEXT_MISSING_ERROR, THREAD_LOCAL_DESTROYED_ERROR}; - /// Blocking pool spawner - pub(super) blocking_spawner: blocking::Spawner, -} +use std::future::Future; +use std::marker::PhantomData; +use std::{error, fmt}; /// Runtime context guard. /// @@ -52,32 +29,37 @@ pub struct Handle { #[derive(Debug)] #[must_use = "Creating and dropping a guard does nothing"] pub struct EnterGuard<'a> { - _guard: context::EnterGuard, + _guard: context::SetCurrentGuard, _handle_lifetime: PhantomData<&'a Handle>, } impl Handle { /// Enters the runtime context. This allows you to construct types that must /// have an executor available on creation such as [`Sleep`] or [`TcpStream`]. - /// It will also allow you to call methods such as [`tokio::spawn`]. + /// It will also allow you to call methods such as [`tokio::spawn`] and [`Handle::current`] + /// without panicking. /// /// [`Sleep`]: struct@crate::time::Sleep /// [`TcpStream`]: struct@crate::net::TcpStream /// [`tokio::spawn`]: fn@crate::spawn pub fn enter(&self) -> EnterGuard<'_> { EnterGuard { - _guard: context::enter(self.clone()), + _guard: match context::try_set_current(&self.inner) { + Some(guard) => guard, + None => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + }, _handle_lifetime: PhantomData, } } /// Returns a `Handle` view over the currently running `Runtime`. /// - /// # Panic + /// # Panics /// /// This will panic if called outside the context of a Tokio runtime. That means that you must - /// call this on one of the threads **being run by the runtime**. Calling this from within a - /// thread created by `std::thread::spawn` (for example) will cause a panic. + /// call this on one of the threads **being run by the runtime**, or from a thread with an active + /// `EnterGuard`. Calling this from within a thread created by `std::thread::spawn` (for example) + /// will cause a panic unless that thread has an active `EnterGuard`. /// /// # Examples /// @@ -101,16 +83,24 @@ impl Handle { /// # let handle = /// thread::spawn(move || { /// // Notice that the handle is created outside of this thread and then moved in - /// handle.spawn(async { /* ... */ }) - /// // This next line would cause a panic - /// // let handle2 = Handle::current(); + /// handle.spawn(async { /* ... */ }); + /// // This next line would cause a panic because we haven't entered the runtime + /// // and created an EnterGuard + /// // let handle2 = Handle::current(); // panic + /// // So we create a guard here with Handle::enter(); + /// let _guard = handle.enter(); + /// // Now we can call Handle::current(); + /// let handle2 = Handle::current(); /// }); /// # handle.join().unwrap(); /// # }); /// # } /// ``` + #[track_caller] pub fn current() -> Self { - context::current() + Handle { + inner: scheduler::Handle::current(), + } } /// Returns a Handle view over the currently running Runtime @@ -119,15 +109,7 @@ impl Handle { /// /// Contrary to `current`, this never panics pub fn try_current() -> Result<Self, TryCurrentError> { - context::try_current() - } - - cfg_stats! { - /// Returns a view that lets you get information about how the runtime - /// is performing. - pub fn stats(&self) -> &crate::runtime::stats::RuntimeStats { - self.spawner.stats() - } + context::try_current().map(|inner| Handle { inner }) } /// Spawns a future onto the Tokio runtime. @@ -136,6 +118,10 @@ impl Handle { /// thread pool. The thread pool is then responsible for polling the future /// until it completes. /// + /// The provided future will start running in the background immediately + /// when `spawn` is called, even if you don't await the returned + /// `JoinHandle`. + /// /// See [module level][mod] documentation for more details. /// /// [mod]: index.html @@ -157,15 +143,13 @@ impl Handle { /// }); /// # } /// ``` - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output> where F: Future + Send + 'static, F::Output: Send + 'static, { - #[cfg(all(tokio_unstable, feature = "tracing"))] - let future = crate::util::trace::task(future, "task", None); - self.spawner.spawn(future) + self.spawn_named(future, None) } /// Runs the provided function on an executor dedicated to blocking. @@ -187,58 +171,13 @@ impl Handle { /// println!("now running on a worker thread"); /// }); /// # } - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R> where F: FnOnce() -> R + Send + 'static, R: Send + 'static, { - if cfg!(debug_assertions) && std::mem::size_of::<F>() > 2048 { - self.spawn_blocking_inner(Box::new(func), None) - } else { - self.spawn_blocking_inner(func, None) - } - } - - #[cfg_attr(tokio_track_caller, track_caller)] - pub(crate) fn spawn_blocking_inner<F, R>(&self, func: F, name: Option<&str>) -> JoinHandle<R> - where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, - { - let fut = BlockingTask::new(func); - - #[cfg(all(tokio_unstable, feature = "tracing"))] - let fut = { - use tracing::Instrument; - #[cfg(tokio_track_caller)] - let location = std::panic::Location::caller(); - #[cfg(tokio_track_caller)] - let span = tracing::trace_span!( - target: "tokio::task::blocking", - "runtime.spawn", - kind = %"blocking", - task.name = %name.unwrap_or_default(), - "fn" = %std::any::type_name::<F>(), - spawn.location = %format_args!("{}:{}:{}", location.file(), location.line(), location.column()), - ); - #[cfg(not(tokio_track_caller))] - let span = tracing::trace_span!( - target: "tokio::task::blocking", - "runtime.spawn", - kind = %"blocking", - task.name = %name.unwrap_or_default(), - "fn" = %std::any::type_name::<F>(), - ); - fut.instrument(span) - }; - - #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let _ = name; - - let (task, handle) = task::unowned(fut, NoopSchedule); - let _ = self.blocking_spawner.spawn(task, self); - handle + self.inner.blocking_spawner().spawn_blocking(self, func) } /// Runs a future to completion on this `Handle`'s associated `Runtime`. @@ -311,25 +250,74 @@ impl Handle { /// [`tokio::fs`]: crate::fs /// [`tokio::net`]: crate::net /// [`tokio::time`]: crate::time - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub fn block_on<F: Future>(&self, future: F) -> F::Output { #[cfg(all(tokio_unstable, feature = "tracing"))] - let future = crate::util::trace::task(future, "block_on", None); - - // Enter the **runtime** context. This configures spawning, the current I/O driver, ... - let _rt_enter = self.enter(); + let future = + crate::util::trace::task(future, "block_on", None, super::task::Id::next().as_u64()); - // Enter a **blocking** context. This prevents blocking from a runtime. - let mut blocking_enter = crate::runtime::enter(true); + // Enter the runtime context. This sets the current driver handles and + // prevents blocking an existing runtime. + let mut enter = context::enter_runtime(&self.inner, true); // Block on the future - blocking_enter + enter + .blocking .block_on(future) .expect("failed to park thread") } - pub(crate) fn shutdown(mut self) { - self.spawner.shutdown(); + #[track_caller] + pub(crate) fn spawn_named<F>(&self, future: F, _name: Option<&str>) -> JoinHandle<F::Output> + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + let id = crate::runtime::task::Id::next(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let future = crate::util::trace::task(future, "task", _name, id.as_u64()); + self.inner.spawn(future, id) + } + + /// Returns the flavor of the current `Runtime`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{Handle, RuntimeFlavor}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() { + /// assert_eq!(RuntimeFlavor::CurrentThread, Handle::current().runtime_flavor()); + /// } + /// ``` + /// + /// ``` + /// use tokio::runtime::{Handle, RuntimeFlavor}; + /// + /// #[tokio::main(flavor = "multi_thread", worker_threads = 4)] + /// async fn main() { + /// assert_eq!(RuntimeFlavor::MultiThread, Handle::current().runtime_flavor()); + /// } + /// ``` + pub fn runtime_flavor(&self) -> RuntimeFlavor { + match self.inner { + scheduler::Handle::CurrentThread(_) => RuntimeFlavor::CurrentThread, + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Handle::MultiThread(_) => RuntimeFlavor::MultiThread, + } + } +} + +cfg_metrics! { + use crate::runtime::RuntimeMetrics; + + impl Handle { + /// Returns a view that lets you get information about how the runtime + /// is performing. + pub fn metrics(&self) -> RuntimeMetrics { + RuntimeMetrics::new(self.clone()) + } } } diff --git a/src/runtime/io/metrics.rs b/src/runtime/io/metrics.rs new file mode 100644 index 0000000..ec341ef --- /dev/null +++ b/src/runtime/io/metrics.rs @@ -0,0 +1,24 @@ +//! This file contains mocks of the metrics types used in the I/O driver. +//! +//! The reason these mocks don't live in `src/runtime/mock.rs` is because +//! these need to be available in the case when `net` is enabled but +//! `rt` is not. + +cfg_not_rt_and_metrics_and_net! { + #[derive(Default)] + pub(crate) struct IoDriverMetrics {} + + impl IoDriverMetrics { + pub(crate) fn incr_fd_count(&self) {} + pub(crate) fn dec_fd_count(&self) {} + pub(crate) fn incr_ready_count_by(&self, _amt: u64) {} + } +} + +cfg_net! { + cfg_rt! { + cfg_metrics! { + pub(crate) use crate::runtime::IoDriverMetrics; + } + } +} diff --git a/src/io/driver/mod.rs b/src/runtime/io/mod.rs index 19f67a2..2e578b6 100644 --- a/src/io/driver/mod.rs +++ b/src/runtime/io/mod.rs @@ -1,12 +1,4 @@ -#![cfg_attr(not(feature = "rt"), allow(dead_code))] - -mod interest; -#[allow(unreachable_pub)] -pub use interest::Interest; - -mod ready; -#[allow(unreachable_pub)] -pub use ready::Ready; +#![cfg_attr(not(all(feature = "rt", feature = "net")), allow(dead_code))] mod registration; pub(crate) use registration::Registration; @@ -14,13 +6,18 @@ pub(crate) use registration::Registration; mod scheduled_io; use scheduled_io::ScheduledIo; -use crate::park::{Park, Unpark}; +mod metrics; + +use crate::io::interest::Interest; +use crate::io::ready::Ready; +use crate::runtime::driver; use crate::util::slab::{self, Slab}; -use crate::{loom::sync::Mutex, util::bit}; +use crate::{loom::sync::RwLock, util::bit}; + +use metrics::IoDriverMetrics; use std::fmt; use std::io; -use std::sync::{Arc, Weak}; use std::time::Duration; /// I/O driver, backed by Mio. @@ -29,51 +26,46 @@ pub(crate) struct Driver { /// as it is mostly used to determine when to call `compact()`. tick: u8, + /// True when an event with the signal token is received + signal_ready: bool, + /// Reuse the `mio::Events` value across calls to poll. - events: Option<mio::Events>, + events: mio::Events, /// Primary slab handle containing the state for each resource registered - /// with this driver. During Drop this is moved into the Inner structure, so - /// this is an Option to allow it to be vacated (until Drop this is always - /// Some). - resources: Option<Slab<ScheduledIo>>, + /// with this driver. + resources: Slab<ScheduledIo>, /// The system event queue. poll: mio::Poll, - - /// State shared between the reactor and the handles. - inner: Arc<Inner>, } /// A reference to an I/O driver. -#[derive(Clone)] pub(crate) struct Handle { - inner: Weak<Inner>, + /// Registers I/O resources. + registry: mio::Registry, + + /// Allocates `ScheduledIo` handles when creating new resources. + io_dispatch: RwLock<IoDispatcher>, + + /// Used to wake up the reactor from a call to `turn`. + /// Not supported on Wasi due to lack of threading support. + #[cfg(not(tokio_wasi))] + waker: mio::Waker, + + pub(crate) metrics: IoDriverMetrics, } #[derive(Debug)] pub(crate) struct ReadyEvent { tick: u8, pub(crate) ready: Ready, + is_shutdown: bool, } -pub(super) struct Inner { - /// Primary slab handle containing the state for each resource registered - /// with this driver. - /// - /// The ownership of this slab is moved into this structure during - /// `Driver::drop`, so that `Inner::drop` can notify all outstanding handles - /// without risking new ones being registered in the meantime. - resources: Mutex<Option<Slab<ScheduledIo>>>, - - /// Registers I/O resources. - registry: mio::Registry, - - /// Allocates `ScheduledIo` handles when creating new resources. - pub(super) io_dispatch: slab::Allocator<ScheduledIo>, - - /// Used to wake up the reactor from a call to `turn`. - waker: mio::Waker, +struct IoDispatcher { + allocator: slab::Allocator<ScheduledIo>, + is_shutdown: bool, } #[derive(Debug, Eq, PartialEq, Clone, Copy)] @@ -90,6 +82,7 @@ enum Tick { // TODO: Don't use a fake token. Instead, reserve a slot entry for the wakeup // token. const TOKEN_WAKEUP: mio::Token = mio::Token(1 << 31); +const TOKEN_SIGNAL: mio::Token = mio::Token(1 + (1 << 31)); const ADDRESS: bit::Pack = bit::Pack::least_significant(24); @@ -112,85 +105,113 @@ fn _assert_kinds() { impl Driver { /// Creates a new event loop, returning any error that happened during the /// creation. - pub(crate) fn new() -> io::Result<Driver> { + pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> { let poll = mio::Poll::new()?; + #[cfg(not(tokio_wasi))] let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?; let registry = poll.registry().try_clone()?; let slab = Slab::new(); let allocator = slab.allocator(); - Ok(Driver { + let driver = Driver { tick: 0, - events: Some(mio::Events::with_capacity(1024)), + signal_ready: false, + events: mio::Events::with_capacity(nevents), poll, - resources: Some(slab), - inner: Arc::new(Inner { - resources: Mutex::new(None), - registry, - io_dispatch: allocator, - waker, - }), - }) + resources: slab, + }; + + let handle = Handle { + registry, + io_dispatch: RwLock::new(IoDispatcher::new(allocator)), + #[cfg(not(tokio_wasi))] + waker, + metrics: IoDriverMetrics::default(), + }; + + Ok((driver, handle)) } - /// Returns a handle to this event loop which can be sent across threads - /// and can be used as a proxy to the event loop itself. - /// - /// Handles are cloneable and clones always refer to the same event loop. - /// This handle is typically passed into functions that create I/O objects - /// to bind them to this event loop. - pub(crate) fn handle(&self) -> Handle { - Handle { - inner: Arc::downgrade(&self.inner), + pub(crate) fn park(&mut self, rt_handle: &driver::Handle) { + let handle = rt_handle.io(); + self.turn(handle, None); + } + + pub(crate) fn park_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) { + let handle = rt_handle.io(); + self.turn(handle, Some(duration)); + } + + pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) { + let handle = rt_handle.io(); + + if handle.shutdown() { + self.resources.for_each(|io| { + // If a task is waiting on the I/O resource, notify it that the + // runtime is being shutdown. And shutdown will clear all wakers. + io.shutdown(); + }); } } - fn turn(&mut self, max_wait: Option<Duration>) -> io::Result<()> { + fn turn(&mut self, handle: &Handle, max_wait: Option<Duration>) { // How often to call `compact()` on the resource slab const COMPACT_INTERVAL: u8 = 255; self.tick = self.tick.wrapping_add(1); if self.tick == COMPACT_INTERVAL { - self.resources.as_mut().unwrap().compact() + self.resources.compact() } - let mut events = self.events.take().expect("i/o driver event store missing"); + let events = &mut self.events; // Block waiting for an event to happen, peeling out how many events // happened. - match self.poll.poll(&mut events, max_wait) { + match self.poll.poll(events, max_wait) { Ok(_) => {} Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), + #[cfg(tokio_wasi)] + Err(e) if e.kind() == io::ErrorKind::InvalidInput => { + // In case of wasm32_wasi this error happens, when trying to poll without subscriptions + // just return from the park, as there would be nothing, which wakes us up. + } + Err(e) => panic!("unexpected error when polling the I/O driver: {:?}", e), } // Process all the events that came in, dispatching appropriately + let mut ready_count = 0; for event in events.iter() { let token = event.token(); - if token != TOKEN_WAKEUP { - self.dispatch(token, Ready::from_mio(event)); + if token == TOKEN_WAKEUP { + // Nothing to do, the event is used to unblock the I/O driver + } else if token == TOKEN_SIGNAL { + self.signal_ready = true; + } else { + Self::dispatch( + &mut self.resources, + self.tick, + token, + Ready::from_mio(event), + ); + ready_count += 1; } } - self.events = Some(events); - - Ok(()) + handle.metrics.incr_ready_count_by(ready_count); } - fn dispatch(&mut self, token: mio::Token, ready: Ready) { + fn dispatch(resources: &mut Slab<ScheduledIo>, tick: u8, token: mio::Token, ready: Ready) { let addr = slab::Address::from_usize(ADDRESS.unpack(token.0)); - let resources = self.resources.as_mut().unwrap(); - let io = match resources.get(addr) { Some(io) => io, None => return, }; - let res = io.set_readiness(Some(token.0), Tick::Set(self.tick), |curr| curr | ready); + let res = io.set_readiness(Some(token.0), Tick::Set(tick), |curr| curr | ready); if res.is_err() { // token no longer valid! @@ -201,84 +222,12 @@ impl Driver { } } -impl Drop for Driver { - fn drop(&mut self) { - (*self.inner.resources.lock()) = self.resources.take(); - } -} - -impl Drop for Inner { - fn drop(&mut self) { - let resources = self.resources.lock().take(); - - if let Some(mut slab) = resources { - slab.for_each(|io| { - // If a task is waiting on the I/O resource, notify it. The task - // will then attempt to use the I/O resource and fail due to the - // driver being shutdown. - io.shutdown(); - }); - } - } -} - -impl Park for Driver { - type Unpark = Handle; - type Error = io::Error; - - fn unpark(&self) -> Self::Unpark { - self.handle() - } - - fn park(&mut self) -> io::Result<()> { - self.turn(None)?; - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> io::Result<()> { - self.turn(Some(duration))?; - Ok(()) - } - - fn shutdown(&mut self) {} -} - impl fmt::Debug for Driver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Driver") } } -// ===== impl Handle ===== - -cfg_rt! { - impl Handle { - /// Returns a handle to the current reactor. - /// - /// # Panics - /// - /// This function panics if there is no current reactor set and `rt` feature - /// flag is not enabled. - pub(super) fn current() -> Self { - crate::runtime::context::io_handle().expect("A Tokio 1.x context was found, but IO is disabled. Call `enable_io` on the runtime builder to enable IO.") - } - } -} - -cfg_not_rt! { - impl Handle { - /// Returns a handle to the current reactor. - /// - /// # Panics - /// - /// This function panics if there is no current reactor set, or if the `rt` - /// feature flag is not enabled. - pub(super) fn current() -> Self { - panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) - } - } -} - impl Handle { /// Forces a reactor blocked in a call to `turn` to wakeup, or otherwise /// makes the next call to `turn` return immediately. @@ -289,32 +238,11 @@ impl Handle { /// after this method has been called. If the reactor is not currently /// blocked in `turn`, then the next call to `turn` will not block and /// return immediately. - fn wakeup(&self) { - if let Some(inner) = self.inner() { - inner.waker.wake().expect("failed to wake I/O driver"); - } - } - - pub(super) fn inner(&self) -> Option<Arc<Inner>> { - self.inner.upgrade() - } -} - -impl Unpark for Handle { - fn unpark(&self) { - self.wakeup(); - } -} - -impl fmt::Debug for Handle { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Handle") + pub(crate) fn unpark(&self) { + #[cfg(not(tokio_wasi))] + self.waker.wake().expect("failed to wake I/O driver"); } -} -// ===== impl Inner ===== - -impl Inner { /// Registers an I/O resource with the reactor for a given `mio::Ready` state. /// /// The registration token is returned. @@ -323,24 +251,68 @@ impl Inner { source: &mut impl mio::event::Source, interest: Interest, ) -> io::Result<slab::Ref<ScheduledIo>> { - let (address, shared) = self.io_dispatch.allocate().ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "reactor at max registered I/O resources", - ) - })?; + let (address, shared) = self.allocate()?; let token = GENERATION.pack(shared.generation(), ADDRESS.pack(address.as_usize(), 0)); self.registry .register(source, mio::Token(token), interest.to_mio())?; + self.metrics.incr_fd_count(); + Ok(shared) } /// Deregisters an I/O resource from the reactor. pub(super) fn deregister_source(&self, source: &mut impl mio::event::Source) -> io::Result<()> { - self.registry.deregister(source) + self.registry.deregister(source)?; + + self.metrics.dec_fd_count(); + + Ok(()) + } + + /// shutdown the dispatcher. + fn shutdown(&self) -> bool { + let mut io = self.io_dispatch.write().unwrap(); + if io.is_shutdown { + return false; + } + io.is_shutdown = true; + true + } + + fn allocate(&self) -> io::Result<(slab::Address, slab::Ref<ScheduledIo>)> { + let io = self.io_dispatch.read().unwrap(); + if io.is_shutdown { + return Err(io::Error::new( + io::ErrorKind::Other, + crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR, + )); + } + io.allocator.allocate().ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "reactor at max registered I/O resources", + ) + }) + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Handle") + } +} + +// ===== impl IoDispatcher ===== + +impl IoDispatcher { + fn new(allocator: slab::Allocator<ScheduledIo>) -> Self { + Self { + allocator, + is_shutdown: false, + } } } @@ -352,3 +324,21 @@ impl Direction { } } } + +// Signal handling +cfg_signal_internal_and_unix! { + impl Handle { + pub(crate) fn register_signal_receiver(&self, receiver: &mut mio::net::UnixStream) -> io::Result<()> { + self.registry.register(receiver, TOKEN_SIGNAL, mio::Interest::READABLE)?; + Ok(()) + } + } + + impl Driver { + pub(crate) fn consume_signal_ready(&mut self) -> bool { + let ret = self.signal_ready; + self.signal_ready = false; + ret + } + } +} diff --git a/src/io/driver/platform.rs b/src/runtime/io/platform.rs index 6b27988..6b27988 100644 --- a/src/io/driver/platform.rs +++ b/src/runtime/io/platform.rs diff --git a/src/io/driver/registration.rs b/src/runtime/io/registration.rs index 7350be6..140b924 100644 --- a/src/io/driver/registration.rs +++ b/src/runtime/io/registration.rs @@ -1,6 +1,8 @@ #![cfg_attr(not(feature = "net"), allow(dead_code))] -use crate::io::driver::{Direction, Handle, Interest, ReadyEvent, ScheduledIo}; +use crate::io::interest::Interest; +use crate::runtime::io::{Direction, Handle, ReadyEvent, ScheduledIo}; +use crate::runtime::scheduler; use crate::util::slab; use mio::event::Source; @@ -42,8 +44,8 @@ cfg_io_driver! { /// [`poll_write_ready`]: method@Self::poll_write_ready` #[derive(Debug)] pub(crate) struct Registration { - /// Handle to the associated driver. - handle: Handle, + /// Handle to the associated runtime. + handle: scheduler::Handle, /// Reference to state stored by the driver. shared: slab::Ref<ScheduledIo>, @@ -56,10 +58,8 @@ unsafe impl Sync for Registration {} // ===== impl Registration ===== impl Registration { - /// Registers the I/O resource with the default reactor, for a specific - /// `Interest`. `new_with_interest` should be used over `new` when you need - /// control over the readiness state, such as when a file descriptor only - /// allows reads. This does not add `hup` or `error` so if you are + /// Registers the I/O resource with the reactor for the provided handle, for + /// a specific `Interest`. This does not add `hup` or `error` so if you are /// interested in those states, you will need to add them to the readiness /// state passed to this function. /// @@ -67,19 +67,13 @@ impl Registration { /// /// - `Ok` if the registration happened successfully /// - `Err` if an error was encountered during registration + #[track_caller] pub(crate) fn new_with_interest_and_handle( io: &mut impl Source, interest: Interest, - handle: Handle, + handle: scheduler::Handle, ) -> io::Result<Registration> { - let shared = if let Some(inner) = handle.inner() { - inner.add_source(io, interest)? - } else { - return Err(io::Error::new( - io::ErrorKind::Other, - "failed to find event loop", - )); - }; + let shared = handle.driver().io().add_source(io, interest)?; Ok(Registration { handle, shared }) } @@ -101,11 +95,7 @@ impl Registration { /// /// `Err` is returned if an error is encountered. pub(crate) fn deregister(&mut self, io: &mut impl Source) -> io::Result<()> { - let inner = match self.handle.inner() { - Some(inner) => inner, - None => return Err(io::Error::new(io::ErrorKind::Other, "reactor gone")), - }; - inner.deregister_source(io) + self.handle().deregister_source(io) } pub(crate) fn clear_readiness(&self, event: ReadyEvent) { @@ -126,6 +116,7 @@ impl Registration { // Uses the poll path, requiring the caller to ensure mutual exclusion for // correctness. Only the last task to call this function is notified. + #[cfg(not(tokio_wasi))] pub(crate) fn poll_read_io<R>( &self, cx: &mut Context<'_>, @@ -154,10 +145,10 @@ impl Registration { direction: Direction, ) -> Poll<io::Result<ReadyEvent>> { // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let ev = ready!(self.shared.poll_readiness(cx, direction)); - if self.handle.inner().is_none() { + if ev.is_shutdown { return Poll::Ready(Err(gone())); } @@ -206,6 +197,10 @@ impl Registration { res => res, } } + + fn handle(&self) -> &Handle { + self.handle.driver().io() + } } impl Drop for Registration { @@ -222,28 +217,22 @@ impl Drop for Registration { } fn gone() -> io::Error { - io::Error::new(io::ErrorKind::Other, "IO driver has terminated") + io::Error::new( + io::ErrorKind::Other, + crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR, + ) } cfg_io_readiness! { impl Registration { pub(crate) async fn readiness(&self, interest: Interest) -> io::Result<ReadyEvent> { - use std::future::Future; - use std::pin::Pin; - - let fut = self.shared.readiness(interest); - pin!(fut); + let ev = self.shared.readiness(interest).await; - crate::future::poll_fn(|cx| { - if self.handle.inner().is_none() { - return Poll::Ready(Err(io::Error::new( - io::ErrorKind::Other, - crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR - ))); - } + if ev.is_shutdown { + return Err(gone()) + } - Pin::new(&mut fut).poll(cx).map(Ok) - }).await + Ok(ev) } pub(crate) async fn async_io<R>(&self, interest: Interest, mut f: impl FnMut() -> io::Result<R>) -> io::Result<R> { diff --git a/src/io/driver/scheduled_io.rs b/src/runtime/io/scheduled_io.rs index 76f9343..197a4e0 100644 --- a/src/io/driver/scheduled_io.rs +++ b/src/runtime/io/scheduled_io.rs @@ -1,4 +1,6 @@ -use super::{Interest, Ready, ReadyEvent, Tick}; +use super::{ReadyEvent, Tick}; +use crate::io::interest::Interest; +use crate::io::ready::Ready; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::Mutex; use crate::util::bit; @@ -44,9 +46,6 @@ struct Waiters { /// Waker used for AsyncWrite. writer: Option<Waker>, - - /// True if this ScheduledIo has been killed due to IO driver shutdown. - is_shutdown: bool, } cfg_io_readiness! { @@ -66,6 +65,14 @@ cfg_io_readiness! { _p: PhantomPinned, } + generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> { + &self.pointers + } + } + } + /// Future returned by `readiness()`. struct Readiness<'a> { scheduled_io: &'a ScheduledIo, @@ -85,7 +92,7 @@ cfg_io_readiness! { // The `ScheduledIo::readiness` (`AtomicUsize`) is packed full of goodness. // -// | reserved | generation | driver tick | readiness | +// | shutdown | generation | driver tick | readiness | // |----------+------------+--------------+-----------| // | 1 bit | 7 bits + 8 bits + 16 bits | @@ -95,6 +102,8 @@ const TICK: bit::Pack = READINESS.then(8); const GENERATION: bit::Pack = TICK.then(7); +const SHUTDOWN: bit::Pack = GENERATION.then(1); + #[test] fn test_generations_assert_same() { assert_eq!(super::GENERATION, GENERATION); @@ -128,9 +137,11 @@ impl ScheduledIo { } /// Invoked when the IO driver is shut down; forces this ScheduledIo into a - /// permanently ready state. + /// permanently shutdown state. pub(super) fn shutdown(&self) { - self.wake0(Ready::ALL, true) + let mask = SHUTDOWN.pack(1, 0); + self.readiness.fetch_or(mask, AcqRel); + self.wake(Ready::ALL); } /// Sets the readiness on this `ScheduledIo` by invoking the given closure on @@ -209,16 +220,10 @@ impl ScheduledIo { /// than 32 wakers to notify, if the stack array fills up, the lock is /// released, the array is cleared, and the iteration continues. pub(super) fn wake(&self, ready: Ready) { - self.wake0(ready, false); - } - - fn wake0(&self, ready: Ready, shutdown: bool) { let mut wakers = WakeList::new(); let mut waiters = self.waiters.lock(); - waiters.is_shutdown |= shutdown; - // check for AsyncRead slot if ready.is_readable() { if let Some(waker) = waiters.reader.take() { @@ -273,6 +278,7 @@ impl ScheduledIo { ReadyEvent { tick: TICK.unpack(curr) as u8, ready: interest.mask() & Ready::from_usize(READINESS.unpack(curr)), + is_shutdown: SHUTDOWN.unpack(curr) != 0, } } @@ -289,8 +295,9 @@ impl ScheduledIo { let curr = self.readiness.load(Acquire); let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr)); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; - if ready.is_empty() { + if ready.is_empty() && !is_shutdown { // Update the task info let mut waiters = self.waiters.lock(); let slot = match direction { @@ -315,10 +322,12 @@ impl ScheduledIo { // taking the waiters lock let curr = self.readiness.load(Acquire); let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr)); - if waiters.is_shutdown { + let is_shutdown = SHUTDOWN.unpack(curr) != 0; + if is_shutdown { Poll::Ready(ReadyEvent { tick: TICK.unpack(curr) as u8, ready: direction.mask(), + is_shutdown, }) } else if ready.is_empty() { Poll::Pending @@ -326,12 +335,14 @@ impl ScheduledIo { Poll::Ready(ReadyEvent { tick: TICK.unpack(curr) as u8, ready, + is_shutdown, }) } } else { Poll::Ready(ReadyEvent { tick: TICK.unpack(curr) as u8, ready, + is_shutdown, }) } } @@ -399,8 +410,8 @@ cfg_io_readiness! { ptr } - unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { + Waiter::addr_of_pointers(target) } } @@ -423,16 +434,17 @@ cfg_io_readiness! { // Optimistically check existing readiness let curr = scheduled_io.readiness.load(SeqCst); let ready = Ready::from_usize(READINESS.unpack(curr)); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; // Safety: `waiter.interest` never changes let interest = unsafe { (*waiter.get()).interest }; let ready = ready.intersection(interest); - if !ready.is_empty() { + if !ready.is_empty() || is_shutdown { // Currently ready! let tick = TICK.unpack(curr) as u8; *state = State::Done; - return Poll::Ready(ReadyEvent { tick, ready }); + return Poll::Ready(ReadyEvent { tick, ready, is_shutdown }); } // Wasn't ready, take the lock (and check again while locked). @@ -440,18 +452,19 @@ cfg_io_readiness! { let curr = scheduled_io.readiness.load(SeqCst); let mut ready = Ready::from_usize(READINESS.unpack(curr)); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; - if waiters.is_shutdown { + if is_shutdown { ready = Ready::ALL; } let ready = ready.intersection(interest); - if !ready.is_empty() { + if !ready.is_empty() || is_shutdown { // Currently ready! let tick = TICK.unpack(curr) as u8; *state = State::Done; - return Poll::Ready(ReadyEvent { tick, ready }); + return Poll::Ready(ReadyEvent { tick, ready, is_shutdown }); } // Not ready even after locked, insert into list... @@ -500,14 +513,26 @@ cfg_io_readiness! { drop(waiters); } State::Done => { - let tick = TICK.unpack(scheduled_io.readiness.load(Acquire)) as u8; - // Safety: State::Done means it is no longer shared let w = unsafe { &mut *waiter.get() }; + let curr = scheduled_io.readiness.load(Acquire); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; + + // The returned tick might be newer than the event + // which notified our waker. This is ok because the future + // still didn't return `Poll::Ready`. + let tick = TICK.unpack(curr) as u8; + + // The readiness state could have been cleared in the meantime, + // but we allow the returned ready set to be empty. + let curr_ready = Ready::from_usize(READINESS.unpack(curr)); + let ready = curr_ready.intersection(w.interest); + return Poll::Ready(ReadyEvent { tick, - ready: Ready::from_interest(w.interest), + ready, + is_shutdown, }); } } diff --git a/src/runtime/metrics/batch.rs b/src/runtime/metrics/batch.rs new file mode 100644 index 0000000..4e6b28d --- /dev/null +++ b/src/runtime/metrics/batch.rs @@ -0,0 +1,116 @@ +use crate::runtime::WorkerMetrics; + +use std::convert::TryFrom; +use std::sync::atomic::Ordering::Relaxed; +use std::time::Instant; + +pub(crate) struct MetricsBatch { + /// Number of times the worker parked. + park_count: u64, + + /// Number of times the worker woke w/o doing work. + noop_count: u64, + + /// Number of tasks stolen. + steal_count: u64, + + /// Number of times tasks where stolen. + steal_operations: u64, + + /// Number of tasks that were polled by the worker. + poll_count: u64, + + /// Number of tasks polled when the worker entered park. This is used to + /// track the noop count. + poll_count_on_last_park: u64, + + /// Number of tasks that were scheduled locally on this worker. + local_schedule_count: u64, + + /// Number of tasks moved to the global queue to make space in the local + /// queue + overflow_count: u64, + + /// The total busy duration in nanoseconds. + busy_duration_total: u64, + last_resume_time: Instant, +} + +impl MetricsBatch { + pub(crate) fn new() -> MetricsBatch { + MetricsBatch { + park_count: 0, + noop_count: 0, + steal_count: 0, + steal_operations: 0, + poll_count: 0, + poll_count_on_last_park: 0, + local_schedule_count: 0, + overflow_count: 0, + busy_duration_total: 0, + last_resume_time: Instant::now(), + } + } + + pub(crate) fn submit(&mut self, worker: &WorkerMetrics) { + worker.park_count.store(self.park_count, Relaxed); + worker.noop_count.store(self.noop_count, Relaxed); + worker.steal_count.store(self.steal_count, Relaxed); + worker + .steal_operations + .store(self.steal_operations, Relaxed); + worker.poll_count.store(self.poll_count, Relaxed); + + worker + .busy_duration_total + .store(self.busy_duration_total, Relaxed); + + worker + .local_schedule_count + .store(self.local_schedule_count, Relaxed); + worker.overflow_count.store(self.overflow_count, Relaxed); + } + + /// The worker is about to park. + pub(crate) fn about_to_park(&mut self) { + self.park_count += 1; + + if self.poll_count_on_last_park == self.poll_count { + self.noop_count += 1; + } else { + self.poll_count_on_last_park = self.poll_count; + } + + let busy_duration = self.last_resume_time.elapsed(); + let busy_duration = u64::try_from(busy_duration.as_nanos()).unwrap_or(u64::MAX); + self.busy_duration_total += busy_duration; + } + + pub(crate) fn returned_from_park(&mut self) { + self.last_resume_time = Instant::now(); + } + + pub(crate) fn inc_local_schedule_count(&mut self) { + self.local_schedule_count += 1; + } + + pub(crate) fn incr_poll_count(&mut self) { + self.poll_count += 1; + } +} + +cfg_rt_multi_thread! { + impl MetricsBatch { + pub(crate) fn incr_steal_count(&mut self, by: u16) { + self.steal_count += by as u64; + } + + pub(crate) fn incr_steal_operations(&mut self) { + self.steal_operations += 1; + } + + pub(crate) fn incr_overflow_count(&mut self) { + self.overflow_count += 1; + } + } +} diff --git a/src/runtime/metrics/io.rs b/src/runtime/metrics/io.rs new file mode 100644 index 0000000..06efdd4 --- /dev/null +++ b/src/runtime/metrics/io.rs @@ -0,0 +1,24 @@ +#![cfg_attr(not(feature = "net"), allow(dead_code))] + +use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +#[derive(Default)] +pub(crate) struct IoDriverMetrics { + pub(super) fd_registered_count: AtomicU64, + pub(super) fd_deregistered_count: AtomicU64, + pub(super) ready_count: AtomicU64, +} + +impl IoDriverMetrics { + pub(crate) fn incr_fd_count(&self) { + self.fd_registered_count.fetch_add(1, Relaxed); + } + + pub(crate) fn dec_fd_count(&self) { + self.fd_deregistered_count.fetch_add(1, Relaxed); + } + + pub(crate) fn incr_ready_count_by(&self, amt: u64) { + self.ready_count.fetch_add(amt, Relaxed); + } +} diff --git a/src/runtime/metrics/mock.rs b/src/runtime/metrics/mock.rs new file mode 100644 index 0000000..c388dc0 --- /dev/null +++ b/src/runtime/metrics/mock.rs @@ -0,0 +1,44 @@ +//! This file contains mocks of the types in src/runtime/metrics + +pub(crate) struct SchedulerMetrics {} + +pub(crate) struct WorkerMetrics {} + +pub(crate) struct MetricsBatch {} + +impl SchedulerMetrics { + pub(crate) fn new() -> Self { + Self {} + } + + /// Increment the number of tasks scheduled externally + pub(crate) fn inc_remote_schedule_count(&self) {} +} + +impl WorkerMetrics { + pub(crate) fn new() -> Self { + Self {} + } + + pub(crate) fn set_queue_depth(&self, _len: usize) {} +} + +impl MetricsBatch { + pub(crate) fn new() -> Self { + Self {} + } + + pub(crate) fn submit(&mut self, _to: &WorkerMetrics) {} + pub(crate) fn about_to_park(&mut self) {} + pub(crate) fn returned_from_park(&mut self) {} + pub(crate) fn incr_poll_count(&mut self) {} + pub(crate) fn inc_local_schedule_count(&mut self) {} +} + +cfg_rt_multi_thread! { + impl MetricsBatch { + pub(crate) fn incr_steal_count(&mut self, _by: u16) {} + pub(crate) fn incr_steal_operations(&mut self) {} + pub(crate) fn incr_overflow_count(&mut self) {} + } +} diff --git a/src/runtime/metrics/mod.rs b/src/runtime/metrics/mod.rs new file mode 100644 index 0000000..4b96f1b --- /dev/null +++ b/src/runtime/metrics/mod.rs @@ -0,0 +1,35 @@ +//! This module contains information need to view information about how the +//! runtime is performing. +//! +//! **Note**: This is an [unstable API][unstable]. The public API of types in +//! this module may break in 1.x releases. See [the documentation on unstable +//! features][unstable] for details. +//! +//! [unstable]: crate#unstable-features +#![allow(clippy::module_inception)] + +cfg_metrics! { + mod batch; + pub(crate) use batch::MetricsBatch; + + mod runtime; + #[allow(unreachable_pub)] // rust-lang/rust#57411 + pub use runtime::RuntimeMetrics; + + mod scheduler; + pub(crate) use scheduler::SchedulerMetrics; + + mod worker; + pub(crate) use worker::WorkerMetrics; + + cfg_net! { + mod io; + pub(crate) use io::IoDriverMetrics; + } +} + +cfg_not_metrics! { + mod mock; + + pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch}; +} diff --git a/src/runtime/metrics/runtime.rs b/src/runtime/metrics/runtime.rs new file mode 100644 index 0000000..d29cb3d --- /dev/null +++ b/src/runtime/metrics/runtime.rs @@ -0,0 +1,658 @@ +use crate::runtime::Handle; + +use std::sync::atomic::Ordering::Relaxed; +use std::time::Duration; + +/// Handle to the runtime's metrics. +/// +/// This handle is internally reference-counted and can be freely cloned. A +/// `RuntimeMetrics` handle is obtained using the [`Runtime::metrics`] method. +/// +/// [`Runtime::metrics`]: crate::runtime::Runtime::metrics() +#[derive(Clone, Debug)] +pub struct RuntimeMetrics { + handle: Handle, +} + +impl RuntimeMetrics { + pub(crate) fn new(handle: Handle) -> RuntimeMetrics { + RuntimeMetrics { handle } + } + + /// Returns the number of worker threads used by the runtime. + /// + /// The number of workers is set by configuring `worker_threads` on + /// `runtime::Builder`. When using the `current_thread` runtime, the return + /// value is always `1`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.num_workers(); + /// println!("Runtime is using {} workers", n); + /// } + /// ``` + pub fn num_workers(&self) -> usize { + self.handle.inner.num_workers() + } + + /// Returns the number of additional threads spawned by the runtime. + /// + /// The number of workers is set by configuring `max_blocking_threads` on + /// `runtime::Builder`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let _ = tokio::task::spawn_blocking(move || { + /// // Stand-in for compute-heavy work or using synchronous APIs + /// 1 + 1 + /// }).await; + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.num_blocking_threads(); + /// println!("Runtime has created {} threads", n); + /// } + /// ``` + pub fn num_blocking_threads(&self) -> usize { + self.handle.inner.num_blocking_threads() + } + + /// Returns the number of idle threads, which hve spawned by the runtime + /// for `spawn_blocking` calls. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let _ = tokio::task::spawn_blocking(move || { + /// // Stand-in for compute-heavy work or using synchronous APIs + /// 1 + 1 + /// }).await; + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.num_idle_blocking_threads(); + /// println!("Runtime has {} idle blocking thread pool threads", n); + /// } + /// ``` + pub fn num_idle_blocking_threads(&self) -> usize { + self.handle.inner.num_idle_blocking_threads() + } + + /// Returns the number of tasks scheduled from **outside** of the runtime. + /// + /// The remote schedule count starts at zero when the runtime is created and + /// increases by one each time a task is woken from **outside** of the + /// runtime. This usually means that a task is spawned or notified from a + /// non-runtime thread and must be queued using the Runtime's injection + /// queue, which tends to be slower. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.remote_schedule_count(); + /// println!("{} tasks were scheduled from outside the runtime", n); + /// } + /// ``` + pub fn remote_schedule_count(&self) -> u64 { + self.handle + .inner + .scheduler_metrics() + .remote_schedule_count + .load(Relaxed) + } + + /// Returns the total number of times the given worker thread has parked. + /// + /// The worker park count starts at zero when the runtime is created and + /// increases by one each time the worker parks the thread waiting for new + /// inbound events to process. This usually means the worker has processed + /// all pending work and is currently idle. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_park_count(0); + /// println!("worker 0 parked {} times", n); + /// } + /// ``` + pub fn worker_park_count(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .park_count + .load(Relaxed) + } + + /// Returns the number of times the given worker thread unparked but + /// performed no work before parking again. + /// + /// The worker no-op count starts at zero when the runtime is created and + /// increases by one each time the worker unparks the thread but finds no + /// new work and goes back to sleep. This indicates a false-positive wake up. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_noop_count(0); + /// println!("worker 0 had {} no-op unparks", n); + /// } + /// ``` + pub fn worker_noop_count(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .noop_count + .load(Relaxed) + } + + /// Returns the number of tasks the given worker thread stole from + /// another worker thread. + /// + /// This metric only applies to the **multi-threaded** runtime and will + /// always return `0` when using the current thread runtime. + /// + /// The worker steal count starts at zero when the runtime is created and + /// increases by `N` each time the worker has processed its scheduled queue + /// and successfully steals `N` more pending tasks from another worker. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_steal_count(0); + /// println!("worker 0 has stolen {} tasks", n); + /// } + /// ``` + pub fn worker_steal_count(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .steal_count + .load(Relaxed) + } + + /// Returns the number of times the given worker thread stole tasks from + /// another worker thread. + /// + /// This metric only applies to the **multi-threaded** runtime and will + /// always return `0` when using the current thread runtime. + /// + /// The worker steal count starts at zero when the runtime is created and + /// increases by one each time the worker has processed its scheduled queue + /// and successfully steals more pending tasks from another worker. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_steal_operations(0); + /// println!("worker 0 has stolen tasks {} times", n); + /// } + /// ``` + pub fn worker_steal_operations(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .steal_operations + .load(Relaxed) + } + + /// Returns the number of tasks the given worker thread has polled. + /// + /// The worker poll count starts at zero when the runtime is created and + /// increases by one each time the worker polls a scheduled task. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_poll_count(0); + /// println!("worker 0 has polled {} tasks", n); + /// } + /// ``` + pub fn worker_poll_count(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .poll_count + .load(Relaxed) + } + + /// Returns the amount of time the given worker thread has been busy. + /// + /// The worker busy duration starts at zero when the runtime is created and + /// increases whenever the worker is spending time processing work. Using + /// this value can indicate the load of the given worker. If a lot of time + /// is spent busy, then the worker is under load and will check for inbound + /// events less often. + /// + /// The timer is monotonically increasing. It is never decremented or reset + /// to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_total_busy_duration(0); + /// println!("worker 0 was busy for a total of {:?}", n); + /// } + /// ``` + pub fn worker_total_busy_duration(&self, worker: usize) -> Duration { + let nanos = self + .handle + .inner + .worker_metrics(worker) + .busy_duration_total + .load(Relaxed); + Duration::from_nanos(nanos) + } + + /// Returns the number of tasks scheduled from **within** the runtime on the + /// given worker's local queue. + /// + /// The local schedule count starts at zero when the runtime is created and + /// increases by one each time a task is woken from **inside** of the + /// runtime on the given worker. This usually means that a task is spawned + /// or notified from within a runtime thread and will be queued on the + /// worker-local queue. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_local_schedule_count(0); + /// println!("{} tasks were scheduled on the worker's local queue", n); + /// } + /// ``` + pub fn worker_local_schedule_count(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .local_schedule_count + .load(Relaxed) + } + + /// Returns the number of times the given worker thread saturated its local + /// queue. + /// + /// This metric only applies to the **multi-threaded** scheduler. + /// + /// The worker steal count starts at zero when the runtime is created and + /// increases by one each time the worker attempts to schedule a task + /// locally, but its local queue is full. When this happens, half of the + /// local queue is moved to the injection queue. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_overflow_count(0); + /// println!("worker 0 has overflowed its queue {} times", n); + /// } + /// ``` + pub fn worker_overflow_count(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .overflow_count + .load(Relaxed) + } + + /// Returns the number of tasks currently scheduled in the runtime's + /// injection queue. + /// + /// Tasks that are spawned or notified from a non-runtime thread are + /// scheduled using the runtime's injection queue. This metric returns the + /// **current** number of tasks pending in the injection queue. As such, the + /// returned value may increase or decrease as new tasks are scheduled and + /// processed. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.injection_queue_depth(); + /// println!("{} tasks currently pending in the runtime's injection queue", n); + /// } + /// ``` + pub fn injection_queue_depth(&self) -> usize { + self.handle.inner.injection_queue_depth() + } + + /// Returns the number of tasks currently scheduled in the given worker's + /// local queue. + /// + /// Tasks that are spawned or notified from within a runtime thread are + /// scheduled using that worker's local queue. This metric returns the + /// **current** number of tasks pending in the worker's local queue. As + /// such, the returned value may increase or decrease as new tasks are + /// scheduled and processed. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_local_queue_depth(0); + /// println!("{} tasks currently pending in worker 0's local queue", n); + /// } + /// ``` + pub fn worker_local_queue_depth(&self, worker: usize) -> usize { + self.handle.inner.worker_local_queue_depth(worker) + } + + /// Returns the number of tasks currently scheduled in the blocking + /// thread pool, spawned using `spawn_blocking`. + /// + /// This metric returns the **current** number of tasks pending in + /// blocking thread pool. As such, the returned value may increase + /// or decrease as new tasks are scheduled and processed. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.blocking_queue_depth(); + /// println!("{} tasks currently pending in the blocking thread pool", n); + /// } + /// ``` + pub fn blocking_queue_depth(&self) -> usize { + self.handle.inner.blocking_queue_depth() + } +} + +cfg_net! { + impl RuntimeMetrics { + /// Returns the number of file descriptors that have been registered with the + /// runtime's I/O driver. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let registered_fds = metrics.io_driver_fd_registered_count(); + /// println!("{} fds have been registered with the runtime's I/O driver.", registered_fds); + /// + /// let deregistered_fds = metrics.io_driver_fd_deregistered_count(); + /// + /// let current_fd_count = registered_fds - deregistered_fds; + /// println!("{} fds are currently registered by the runtime's I/O driver.", current_fd_count); + /// } + /// ``` + pub fn io_driver_fd_registered_count(&self) -> u64 { + self.with_io_driver_metrics(|m| { + m.fd_registered_count.load(Relaxed) + }) + } + + /// Returns the number of file descriptors that have been deregistered by the + /// runtime's I/O driver. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.io_driver_fd_deregistered_count(); + /// println!("{} fds have been deregistered by the runtime's I/O driver.", n); + /// } + /// ``` + pub fn io_driver_fd_deregistered_count(&self) -> u64 { + self.with_io_driver_metrics(|m| { + m.fd_deregistered_count.load(Relaxed) + }) + } + + /// Returns the number of ready events processed by the runtime's + /// I/O driver. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.io_driver_ready_count(); + /// println!("{} ready events processed by the runtime's I/O driver.", n); + /// } + /// ``` + pub fn io_driver_ready_count(&self) -> u64 { + self.with_io_driver_metrics(|m| m.ready_count.load(Relaxed)) + } + + fn with_io_driver_metrics<F>(&self, f: F) -> u64 + where + F: Fn(&super::IoDriverMetrics) -> u64, + { + // TODO: Investigate if this should return 0, most of our metrics always increase + // thus this breaks that guarantee. + self.handle + .inner + .driver() + .io + .as_ref() + .map(|h| f(&h.metrics)) + .unwrap_or(0) + } + } +} diff --git a/src/runtime/metrics/scheduler.rs b/src/runtime/metrics/scheduler.rs new file mode 100644 index 0000000..d1ba3b6 --- /dev/null +++ b/src/runtime/metrics/scheduler.rs @@ -0,0 +1,27 @@ +use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +/// Retrieves metrics from the Tokio runtime. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [unstable]: crate#unstable-features +#[derive(Debug)] +pub(crate) struct SchedulerMetrics { + /// Number of tasks that are scheduled from outside the runtime. + pub(super) remote_schedule_count: AtomicU64, +} + +impl SchedulerMetrics { + pub(crate) fn new() -> SchedulerMetrics { + SchedulerMetrics { + remote_schedule_count: AtomicU64::new(0), + } + } + + /// Increment the number of tasks scheduled externally + pub(crate) fn inc_remote_schedule_count(&self) { + self.remote_schedule_count.fetch_add(1, Relaxed); + } +} diff --git a/src/runtime/metrics/worker.rs b/src/runtime/metrics/worker.rs new file mode 100644 index 0000000..a40c76e --- /dev/null +++ b/src/runtime/metrics/worker.rs @@ -0,0 +1,65 @@ +use crate::loom::sync::atomic::Ordering::Relaxed; +use crate::loom::sync::atomic::{AtomicU64, AtomicUsize}; + +/// Retrieve runtime worker metrics. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [unstable]: crate#unstable-features +#[derive(Debug)] +#[repr(align(128))] +pub(crate) struct WorkerMetrics { + /// Number of times the worker parked. + pub(crate) park_count: AtomicU64, + + /// Number of times the worker woke then parked again without doing work. + pub(crate) noop_count: AtomicU64, + + /// Number of tasks the worker stole. + pub(crate) steal_count: AtomicU64, + + /// Number of times the worker stole + pub(crate) steal_operations: AtomicU64, + + /// Number of tasks the worker polled. + pub(crate) poll_count: AtomicU64, + + /// Amount of time the worker spent doing work vs. parking. + pub(crate) busy_duration_total: AtomicU64, + + /// Number of tasks scheduled for execution on the worker's local queue. + pub(crate) local_schedule_count: AtomicU64, + + /// Number of tasks moved from the local queue to the global queue to free space. + pub(crate) overflow_count: AtomicU64, + + /// Number of tasks currently in the local queue. Used only by the + /// current-thread scheduler. + pub(crate) queue_depth: AtomicUsize, +} + +impl WorkerMetrics { + pub(crate) fn new() -> WorkerMetrics { + WorkerMetrics { + park_count: AtomicU64::new(0), + noop_count: AtomicU64::new(0), + steal_count: AtomicU64::new(0), + steal_operations: AtomicU64::new(0), + poll_count: AtomicU64::new(0), + overflow_count: AtomicU64::new(0), + busy_duration_total: AtomicU64::new(0), + local_schedule_count: AtomicU64::new(0), + queue_depth: AtomicUsize::new(0), + } + } + + pub(crate) fn queue_depth(&self) -> usize { + self.queue_depth.load(Relaxed) + } + + pub(crate) fn set_queue_depth(&self, len: usize) { + self.queue_depth.store(len, Relaxed); + } +} diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index 96bb47c..b6f43ea 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -137,7 +137,7 @@ //! use tokio::runtime; //! //! # fn main() -> Result<(), Box<dyn std::error::Error>> { -//! let basic_rt = runtime::Builder::new_current_thread() +//! let rt = runtime::Builder::new_current_thread() //! .build()?; //! # Ok(()) } //! ``` @@ -166,7 +166,6 @@ //! [`tokio::main`]: ../attr.main.html //! [runtime builder]: crate::runtime::Builder //! [`Runtime::new`]: crate::runtime::Runtime::new -//! [`Builder::basic_scheduler`]: crate::runtime::Builder::basic_scheduler //! [`Builder::threaded_scheduler`]: crate::runtime::Builder::threaded_scheduler //! [`Builder::enable_io`]: crate::runtime::Builder::enable_io //! [`Builder::enable_time`]: crate::runtime::Builder::enable_time @@ -174,427 +173,89 @@ // At the top due to macros #[cfg(test)] +#[cfg(not(tokio_wasm))] #[macro_use] mod tests; -pub(crate) mod enter; +pub(crate) mod context; -pub(crate) mod task; +pub(crate) mod coop; -cfg_stats! { - pub mod stats; -} -cfg_not_stats! { - pub(crate) mod stats; -} - -cfg_rt! { - mod basic_scheduler; - use basic_scheduler::BasicScheduler; - - mod blocking; - use blocking::BlockingPool; - pub(crate) use blocking::spawn_blocking; - - mod builder; - pub use self::builder::Builder; +pub(crate) mod park; - pub(crate) mod context; - pub(crate) mod driver; +mod driver; - use self::enter::enter; +pub(crate) mod scheduler; - mod handle; - pub use handle::{EnterGuard, Handle, TryCurrentError}; - - mod spawner; - use self::spawner::Spawner; +cfg_io_driver_impl! { + pub(crate) mod io; } -cfg_rt_multi_thread! { - mod park; - use park::Parker; +cfg_process_driver! { + mod process; } -cfg_rt_multi_thread! { - mod queue; +cfg_time! { + pub(crate) mod time; +} - pub(crate) mod thread_pool; - use self::thread_pool::ThreadPool; +cfg_signal_internal_and_unix! { + pub(crate) mod signal; } cfg_rt! { - use crate::task::JoinHandle; - - use std::future::Future; - use std::time::Duration; + pub(crate) mod task; - /// The Tokio runtime. - /// - /// The runtime provides an I/O driver, task scheduler, [timer], and - /// blocking pool, necessary for running asynchronous tasks. - /// - /// Instances of `Runtime` can be created using [`new`], or [`Builder`]. - /// However, most users will use the `#[tokio::main]` annotation on their - /// entry point instead. - /// - /// See [module level][mod] documentation for more details. - /// - /// # Shutdown - /// - /// Shutting down the runtime is done by dropping the value. The current - /// thread will block until the shut down operation has completed. - /// - /// * Drain any scheduled work queues. - /// * Drop any futures that have not yet completed. - /// * Drop the reactor. - /// - /// Once the reactor has dropped, any outstanding I/O resources bound to - /// that reactor will no longer function. Calling any method on them will - /// result in an error. - /// - /// # Sharing - /// - /// The Tokio runtime implements `Sync` and `Send` to allow you to wrap it - /// in a `Arc`. Most fn take `&self` to allow you to call them concurrently - /// across multiple threads. - /// - /// Calls to `shutdown` and `shutdown_timeout` require exclusive ownership of - /// the runtime type and this can be achieved via `Arc::try_unwrap` when only - /// one strong count reference is left over. - /// - /// [timer]: crate::time - /// [mod]: index.html - /// [`new`]: method@Self::new - /// [`Builder`]: struct@Builder - #[derive(Debug)] - pub struct Runtime { - /// Task executor - kind: Kind, + mod config; + use config::Config; - /// Handle to runtime, also contains driver handles - handle: Handle, + mod blocking; + #[cfg_attr(tokio_wasi, allow(unused_imports))] + pub(crate) use blocking::spawn_blocking; - /// Blocking pool handle, used to signal shutdown - blocking_pool: BlockingPool, + cfg_trace! { + pub(crate) use blocking::Mandatory; } - /// The runtime executor is either a thread-pool or a current-thread executor. - #[derive(Debug)] - enum Kind { - /// Execute all tasks on the current-thread. - CurrentThread(BasicScheduler<driver::Driver>), - - /// Execute tasks across multiple threads. - #[cfg(feature = "rt-multi-thread")] - ThreadPool(ThreadPool), + cfg_fs! { + pub(crate) use blocking::spawn_mandatory_blocking; } - /// After thread starts / before thread stops - type Callback = std::sync::Arc<dyn Fn() + Send + Sync>; - - impl Runtime { - /// Creates a new runtime instance with default configuration values. - /// - /// This results in the multi threaded scheduler, I/O driver, and time driver being - /// initialized. - /// - /// Most applications will not need to call this function directly. Instead, - /// they will use the [`#[tokio::main]` attribute][main]. When a more complex - /// configuration is necessary, the [runtime builder] may be used. - /// - /// See [module level][mod] documentation for more details. - /// - /// # Examples - /// - /// Creating a new `Runtime` with default configuration values. - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// let rt = Runtime::new() - /// .unwrap(); - /// - /// // Use the runtime... - /// ``` - /// - /// [mod]: index.html - /// [main]: ../attr.main.html - /// [threaded scheduler]: index.html#threaded-scheduler - /// [basic scheduler]: index.html#basic-scheduler - /// [runtime builder]: crate::runtime::Builder - #[cfg(feature = "rt-multi-thread")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] - pub fn new() -> std::io::Result<Runtime> { - Builder::new_multi_thread().enable_all().build() - } - - /// Returns a handle to the runtime's spawner. - /// - /// The returned handle can be used to spawn tasks that run on this runtime, and can - /// be cloned to allow moving the `Handle` to other threads. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// let rt = Runtime::new() - /// .unwrap(); - /// - /// let handle = rt.handle(); - /// - /// // Use the handle... - /// ``` - pub fn handle(&self) -> &Handle { - &self.handle - } - - /// Spawns a future onto the Tokio runtime. - /// - /// This spawns the given future onto the runtime's executor, usually a - /// thread pool. The thread pool is then responsible for polling the future - /// until it completes. - /// - /// See [module level][mod] documentation for more details. - /// - /// [mod]: index.html - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// # fn dox() { - /// // Create the runtime - /// let rt = Runtime::new().unwrap(); - /// - /// // Spawn a future onto the runtime - /// rt.spawn(async { - /// println!("now running on a worker thread"); - /// }); - /// # } - /// ``` - #[cfg_attr(tokio_track_caller, track_caller)] - pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output> - where - F: Future + Send + 'static, - F::Output: Send + 'static, - { - self.handle.spawn(future) - } + mod builder; + pub use self::builder::Builder; + cfg_unstable! { + pub use self::builder::UnhandledPanic; + pub use crate::util::rand::RngSeed; + } - /// Runs the provided function on an executor dedicated to blocking operations. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// # fn dox() { - /// // Create the runtime - /// let rt = Runtime::new().unwrap(); - /// - /// // Spawn a blocking function onto the runtime - /// rt.spawn_blocking(|| { - /// println!("now running on a worker thread"); - /// }); - /// # } - #[cfg_attr(tokio_track_caller, track_caller)] - pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R> - where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, - { - self.handle.spawn_blocking(func) - } + mod defer; + pub(crate) use defer::Defer; - /// Runs a future to completion on the Tokio runtime. This is the - /// runtime's entry point. - /// - /// This runs the given future on the current thread, blocking until it is - /// complete, and yielding its resolved result. Any tasks or timers - /// which the future spawns internally will be executed on the runtime. - /// - /// # Multi thread scheduler - /// - /// When the multi thread scheduler is used this will allow futures - /// to run within the io driver and timer context of the overall runtime. - /// - /// # Current thread scheduler - /// - /// When the current thread scheduler is enabled `block_on` - /// can be called concurrently from multiple threads. The first call - /// will take ownership of the io and timer drivers. This means - /// other threads which do not own the drivers will hook into that one. - /// When the first `block_on` completes, other threads will be able to - /// "steal" the driver to allow continued execution of their futures. - /// - /// # Panics - /// - /// This function panics if the provided future panics, or if called within an - /// asynchronous execution context. - /// - /// # Examples - /// - /// ```no_run - /// use tokio::runtime::Runtime; - /// - /// // Create the runtime - /// let rt = Runtime::new().unwrap(); - /// - /// // Execute the future, blocking the current thread until completion - /// rt.block_on(async { - /// println!("hello"); - /// }); - /// ``` - /// - /// [handle]: fn@Handle::block_on - #[cfg_attr(tokio_track_caller, track_caller)] - pub fn block_on<F: Future>(&self, future: F) -> F::Output { - #[cfg(all(tokio_unstable, feature = "tracing"))] - let future = crate::util::trace::task(future, "block_on", None); + mod handle; + pub use handle::{EnterGuard, Handle, TryCurrentError}; - let _enter = self.enter(); + mod runtime; + pub use runtime::{Runtime, RuntimeFlavor}; - match &self.kind { - Kind::CurrentThread(exec) => exec.block_on(future), - #[cfg(feature = "rt-multi-thread")] - Kind::ThreadPool(exec) => exec.block_on(future), - } - } + mod thread_id; + pub(crate) use thread_id::ThreadId; - /// Enters the runtime context. - /// - /// This allows you to construct types that must have an executor - /// available on creation such as [`Sleep`] or [`TcpStream`]. It will - /// also allow you to call methods such as [`tokio::spawn`]. - /// - /// [`Sleep`]: struct@crate::time::Sleep - /// [`TcpStream`]: struct@crate::net::TcpStream - /// [`tokio::spawn`]: fn@crate::spawn - /// - /// # Example - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// fn function_that_spawns(msg: String) { - /// // Had we not used `rt.enter` below, this would panic. - /// tokio::spawn(async move { - /// println!("{}", msg); - /// }); - /// } - /// - /// fn main() { - /// let rt = Runtime::new().unwrap(); - /// - /// let s = "Hello World!".to_string(); - /// - /// // By entering the context, we tie `tokio::spawn` to this executor. - /// let _guard = rt.enter(); - /// function_that_spawns(s); - /// } - /// ``` - pub fn enter(&self) -> EnterGuard<'_> { - self.handle.enter() - } + cfg_metrics! { + mod metrics; + pub use metrics::RuntimeMetrics; - /// Shuts down the runtime, waiting for at most `duration` for all spawned - /// task to shutdown. - /// - /// Usually, dropping a `Runtime` handle is sufficient as tasks are able to - /// shutdown in a timely fashion. However, dropping a `Runtime` will wait - /// indefinitely for all tasks to terminate, and there are cases where a long - /// blocking task has been spawned, which can block dropping `Runtime`. - /// - /// In this case, calling `shutdown_timeout` with an explicit wait timeout - /// can work. The `shutdown_timeout` will signal all tasks to shutdown and - /// will wait for at most `duration` for all spawned tasks to terminate. If - /// `timeout` elapses before all tasks are dropped, the function returns and - /// outstanding tasks are potentially leaked. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// use tokio::task; - /// - /// use std::thread; - /// use std::time::Duration; - /// - /// fn main() { - /// let runtime = Runtime::new().unwrap(); - /// - /// runtime.block_on(async move { - /// task::spawn_blocking(move || { - /// thread::sleep(Duration::from_secs(10_000)); - /// }); - /// }); - /// - /// runtime.shutdown_timeout(Duration::from_millis(100)); - /// } - /// ``` - pub fn shutdown_timeout(mut self, duration: Duration) { - // Wakeup and shutdown all the worker threads - self.handle.clone().shutdown(); - self.blocking_pool.shutdown(Some(duration)); - } + pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics}; - /// Shuts down the runtime, without waiting for any spawned tasks to shutdown. - /// - /// This can be useful if you want to drop a runtime from within another runtime. - /// Normally, dropping a runtime will block indefinitely for spawned blocking tasks - /// to complete, which would normally not be permitted within an asynchronous context. - /// By calling `shutdown_background()`, you can drop the runtime from such a context. - /// - /// Note however, that because we do not wait for any blocking tasks to complete, this - /// may result in a resource leak (in that any blocking tasks are still running until they - /// return. - /// - /// This function is equivalent to calling `shutdown_timeout(Duration::of_nanos(0))`. - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// fn main() { - /// let runtime = Runtime::new().unwrap(); - /// - /// runtime.block_on(async move { - /// let inner_runtime = Runtime::new().unwrap(); - /// // ... - /// inner_runtime.shutdown_background(); - /// }); - /// } - /// ``` - pub fn shutdown_background(self) { - self.shutdown_timeout(Duration::from_nanos(0)) + cfg_net! { + pub(crate) use metrics::IoDriverMetrics; } } - #[allow(clippy::single_match)] // there are comments in the error branch, so we don't want if-let - impl Drop for Runtime { - fn drop(&mut self) { - match &mut self.kind { - Kind::CurrentThread(basic) => { - // This ensures that tasks spawned on the basic runtime are dropped inside the - // runtime's context. - match self::context::try_enter(self.handle.clone()) { - Some(guard) => basic.set_context_guard(guard), - None => { - // The context thread-local has alread been destroyed. - // - // We don't set the guard in this case. Calls to tokio::spawn in task - // destructors would fail regardless if this happens. - }, - } - }, - #[cfg(feature = "rt-multi-thread")] - Kind::ThreadPool(_) => { - // The threaded scheduler drops its tasks on its worker threads, which is - // already in the runtime's context. - }, - } - } + cfg_not_metrics! { + pub(crate) mod metrics; + pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch}; } + + /// After thread starts / before thread stops + type Callback = std::sync::Arc<dyn Fn() + Send + Sync>; } diff --git a/src/runtime/park.rs b/src/runtime/park.rs index 033b9f2..dc86f42 100644 --- a/src/runtime/park.rs +++ b/src/runtime/park.rs @@ -1,153 +1,102 @@ -//! Parks the runtime. -//! -//! A combination of the various resource driver park handles. +#![cfg_attr(not(feature = "full"), allow(dead_code))] use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::{Arc, Condvar, Mutex}; -use crate::loom::thread; -use crate::park::{Park, Unpark}; -use crate::runtime::driver::Driver; -use crate::util::TryLock; use std::sync::atomic::Ordering::SeqCst; use std::time::Duration; -pub(crate) struct Parker { +#[derive(Debug)] +pub(crate) struct ParkThread { inner: Arc<Inner>, } -pub(crate) struct Unparker { +/// Unblocks a thread that was blocked by `ParkThread`. +#[derive(Clone, Debug)] +pub(crate) struct UnparkThread { inner: Arc<Inner>, } +#[derive(Debug)] struct Inner { - /// Avoids entering the park if possible state: AtomicUsize, - - /// Used to coordinate access to the driver / condvar mutex: Mutex<()>, - - /// Condvar to block on if the driver is unavailable. condvar: Condvar, - - /// Resource (I/O, time, ...) driver - shared: Arc<Shared>, } const EMPTY: usize = 0; -const PARKED_CONDVAR: usize = 1; -const PARKED_DRIVER: usize = 2; -const NOTIFIED: usize = 3; +const PARKED: usize = 1; +const NOTIFIED: usize = 2; -/// Shared across multiple Parker handles -struct Shared { - /// Shared driver. Only one thread at a time can use this - driver: TryLock<Driver>, +tokio_thread_local! { + static CURRENT_PARKER: ParkThread = ParkThread::new(); +} - /// Unpark handle - handle: <Driver as Park>::Unpark, +// Bit of a hack, but it is only for loom +#[cfg(loom)] +tokio_thread_local! { + static CURRENT_THREAD_PARK_COUNT: AtomicUsize = AtomicUsize::new(0); } -impl Parker { - pub(crate) fn new(driver: Driver) -> Parker { - let handle = driver.unpark(); +// ==== impl ParkThread ==== - Parker { +impl ParkThread { + pub(crate) fn new() -> Self { + Self { inner: Arc::new(Inner { state: AtomicUsize::new(EMPTY), mutex: Mutex::new(()), condvar: Condvar::new(), - shared: Arc::new(Shared { - driver: TryLock::new(driver), - handle, - }), }), } } -} -impl Clone for Parker { - fn clone(&self) -> Parker { - Parker { - inner: Arc::new(Inner { - state: AtomicUsize::new(EMPTY), - mutex: Mutex::new(()), - condvar: Condvar::new(), - shared: self.inner.shared.clone(), - }), - } + pub(crate) fn unpark(&self) -> UnparkThread { + let inner = self.inner.clone(); + UnparkThread { inner } } -} - -impl Park for Parker { - type Unpark = Unparker; - type Error = (); - fn unpark(&self) -> Unparker { - Unparker { - inner: self.inner.clone(), - } - } - - fn park(&mut self) -> Result<(), Self::Error> { + pub(crate) fn park(&mut self) { + #[cfg(loom)] + CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst)); self.inner.park(); - Ok(()) } - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - // Only parking with zero is supported... - assert_eq!(duration, Duration::from_millis(0)); + pub(crate) fn park_timeout(&mut self, duration: Duration) { + #[cfg(loom)] + CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst)); - if let Some(mut driver) = self.inner.shared.driver.try_lock() { - driver.park_timeout(duration).map_err(|_| ()) - } else { - Ok(()) - } + // Wasm doesn't have threads, so just sleep. + #[cfg(not(tokio_wasm))] + self.inner.park_timeout(duration); + #[cfg(tokio_wasm)] + std::thread::sleep(duration); } - fn shutdown(&mut self) { + pub(crate) fn shutdown(&mut self) { self.inner.shutdown(); } } -impl Unpark for Unparker { - fn unpark(&self) { - self.inner.unpark(); - } -} +// ==== impl Inner ==== impl Inner { /// Parks the current thread for at most `dur`. fn park(&self) { - for _ in 0..3 { - // If we were previously notified then we consume this notification and - // return quickly. - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - return; - } - - thread::yield_now(); - } - - if let Some(mut driver) = self.shared.driver.try_lock() { - self.park_driver(&mut driver); - } else { - self.park_condvar(); + // If we were previously notified then we consume this notification and + // return quickly. + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + return; } - } - fn park_condvar(&self) { // Otherwise we need to coordinate going to sleep let mut m = self.mutex.lock(); - match self - .state - .compare_exchange(EMPTY, PARKED_CONDVAR, SeqCst, SeqCst) - { + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { Ok(_) => {} Err(NOTIFIED) => { // We must read here, even though we know it will be `NOTIFIED`. @@ -180,33 +129,44 @@ impl Inner { } } - fn park_driver(&self, driver: &mut Driver) { - match self + fn park_timeout(&self, dur: Duration) { + // Like `park` above we have a fast path for an already-notified thread, + // and afterwards we start coordinating for a sleep. Return quickly. + if self .state - .compare_exchange(EMPTY, PARKED_DRIVER, SeqCst, SeqCst) + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() { + return; + } + + if dur == Duration::from_millis(0) { + return; + } + + let m = self.mutex.lock(); + + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { Ok(_) => {} Err(NOTIFIED) => { - // We must read here, even though we know it will be `NOTIFIED`. - // This is because `unpark` may have been called again since we read - // `NOTIFIED` in the `compare_exchange` above. We must perform an - // acquire operation that synchronizes with that `unpark` to observe - // any writes it made before the call to unpark. To do that we must - // read from the write it made to `state`. + // We must read again here, see `park`. let old = self.state.swap(EMPTY, SeqCst); debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); return; } - Err(actual) => panic!("inconsistent park state; actual = {}", actual), + Err(actual) => panic!("inconsistent park_timeout state; actual = {}", actual), } - // TODO: don't unwrap - driver.park().unwrap(); + // Wait with a timeout, and if we spuriously wake up or otherwise wake up + // from a notification, we just want to unconditionally set the state back to + // empty, either consuming a notification or un-flagging ourselves as + // parked. + let (_m, _result) = self.condvar.wait_timeout(m, dur).unwrap(); match self.state.swap(EMPTY, SeqCst) { - NOTIFIED => {} // got a notification, hurray! - PARKED_DRIVER => {} // no notification, alas + NOTIFIED => {} // got a notification, hurray! + PARKED => {} // no notification, alas n => panic!("inconsistent park_timeout state: {}", n), } } @@ -218,15 +178,12 @@ impl Inner { // is already `NOTIFIED`. That is why this must be a swap rather than a // compare-and-swap that returns if it reads `NOTIFIED` on failure. match self.state.swap(NOTIFIED, SeqCst) { - EMPTY => {} // no one was waiting - NOTIFIED => {} // already unparked - PARKED_CONDVAR => self.unpark_condvar(), - PARKED_DRIVER => self.unpark_driver(), - actual => panic!("inconsistent state in unpark; actual = {}", actual), + EMPTY => return, // no one was waiting + NOTIFIED => return, // already unparked + PARKED => {} // gotta go wake someone up + _ => panic!("inconsistent state in unpark"), } - } - fn unpark_condvar(&self) { // There is a period between when the parked thread sets `state` to // `PARKED` (or last checked `state` in the case of a spurious wake // up) and when it actually waits on `cvar`. If we were to notify @@ -243,15 +200,154 @@ impl Inner { self.condvar.notify_one() } - fn unpark_driver(&self) { - self.shared.handle.unpark(); + fn shutdown(&self) { + self.condvar.notify_all(); } +} - fn shutdown(&self) { - if let Some(mut driver) = self.shared.driver.try_lock() { - driver.shutdown(); +impl Default for ParkThread { + fn default() -> Self { + Self::new() + } +} + +// ===== impl UnparkThread ===== + +impl UnparkThread { + pub(crate) fn unpark(&self) { + self.inner.unpark(); + } +} + +use crate::loom::thread::AccessError; +use std::future::Future; +use std::marker::PhantomData; +use std::mem; +use std::rc::Rc; +use std::task::{RawWaker, RawWakerVTable, Waker}; + +/// Blocks the current thread using a condition variable. +#[derive(Debug)] +pub(crate) struct CachedParkThread { + _anchor: PhantomData<Rc<()>>, +} + +impl CachedParkThread { + /// Creates a new `ParkThread` handle for the current thread. + /// + /// This type cannot be moved to other threads, so it should be created on + /// the thread that the caller intends to park. + pub(crate) fn new() -> CachedParkThread { + CachedParkThread { + _anchor: PhantomData, } + } - self.condvar.notify_all(); + pub(crate) fn waker(&self) -> Result<Waker, AccessError> { + self.unpark().map(|unpark| unpark.into_waker()) + } + + fn unpark(&self) -> Result<UnparkThread, AccessError> { + self.with_current(|park_thread| park_thread.unpark()) + } + + pub(crate) fn park(&mut self) { + self.with_current(|park_thread| park_thread.inner.park()) + .unwrap(); + } + + pub(crate) fn park_timeout(&mut self, duration: Duration) { + self.with_current(|park_thread| park_thread.inner.park_timeout(duration)) + .unwrap(); + } + + /// Gets a reference to the `ParkThread` handle for this thread. + fn with_current<F, R>(&self, f: F) -> Result<R, AccessError> + where + F: FnOnce(&ParkThread) -> R, + { + CURRENT_PARKER.try_with(|inner| f(inner)) + } + + pub(crate) fn block_on<F: Future>(&mut self, f: F) -> Result<F::Output, AccessError> { + use std::task::Context; + use std::task::Poll::Ready; + + // `get_unpark()` should not return a Result + let waker = self.waker()?; + let mut cx = Context::from_waker(&waker); + + pin!(f); + + loop { + if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) { + return Ok(v); + } + + // Wake any yielded tasks before parking in order to avoid + // blocking. + #[cfg(feature = "rt")] + crate::runtime::context::with_defer(|defer| defer.wake()); + + self.park(); + } + } +} + +impl UnparkThread { + pub(crate) fn into_waker(self) -> Waker { + unsafe { + let raw = unparker_to_raw_waker(self.inner); + Waker::from_raw(raw) + } + } +} + +impl Inner { + #[allow(clippy::wrong_self_convention)] + fn into_raw(this: Arc<Inner>) -> *const () { + Arc::into_raw(this) as *const () } + + unsafe fn from_raw(ptr: *const ()) -> Arc<Inner> { + Arc::from_raw(ptr as *const Inner) + } +} + +unsafe fn unparker_to_raw_waker(unparker: Arc<Inner>) -> RawWaker { + RawWaker::new( + Inner::into_raw(unparker), + &RawWakerVTable::new(clone, wake, wake_by_ref, drop_waker), + ) +} + +unsafe fn clone(raw: *const ()) -> RawWaker { + let unparker = Inner::from_raw(raw); + + // Increment the ref count + mem::forget(unparker.clone()); + + unparker_to_raw_waker(unparker) +} + +unsafe fn drop_waker(raw: *const ()) { + let _ = Inner::from_raw(raw); +} + +unsafe fn wake(raw: *const ()) { + let unparker = Inner::from_raw(raw); + unparker.unpark(); +} + +unsafe fn wake_by_ref(raw: *const ()) { + let unparker = Inner::from_raw(raw); + unparker.unpark(); + + // We don't actually own a reference to the unparker + mem::forget(unparker); +} + +#[cfg(loom)] +pub(crate) fn current_thread_park_count() -> usize { + CURRENT_THREAD_PARK_COUNT.with(|count| count.load(SeqCst)) } diff --git a/src/process/unix/driver.rs b/src/runtime/process.rs index 84dc8fb..df339b0 100644 --- a/src/process/unix/driver.rs +++ b/src/runtime/process.rs @@ -2,11 +2,10 @@ //! Process driver. -use crate::park::Park; use crate::process::unix::GlobalOrphanQueue; -use crate::signal::unix::driver::{Driver as SignalDriver, Handle as SignalHandle}; +use crate::runtime::driver; +use crate::runtime::signal::{Driver as SignalDriver, Handle as SignalHandle}; -use std::io; use std::time::Duration; /// Responsible for cleaning up orphaned child processes on Unix platforms. @@ -28,31 +27,18 @@ impl Driver { signal_handle, } } -} - -// ===== impl Park for Driver ===== - -impl Park for Driver { - type Unpark = <SignalDriver as Park>::Unpark; - type Error = io::Error; - - fn unpark(&self) -> Self::Unpark { - self.park.unpark() - } - fn park(&mut self) -> Result<(), Self::Error> { - self.park.park()?; + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.park.park(handle); GlobalOrphanQueue::reap_orphans(&self.signal_handle); - Ok(()) } - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.park.park_timeout(duration)?; + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + self.park.park_timeout(handle, duration); GlobalOrphanQueue::reap_orphans(&self.signal_handle); - Ok(()) } - fn shutdown(&mut self) { - self.park.shutdown() + pub(crate) fn shutdown(&mut self, handle: &driver::Handle) { + self.park.shutdown(handle) } } diff --git a/src/runtime/runtime.rs b/src/runtime/runtime.rs new file mode 100644 index 0000000..1985673 --- /dev/null +++ b/src/runtime/runtime.rs @@ -0,0 +1,423 @@ +use crate::runtime::blocking::BlockingPool; +use crate::runtime::scheduler::CurrentThread; +use crate::runtime::{context, EnterGuard, Handle}; +use crate::task::JoinHandle; + +use std::future::Future; +use std::time::Duration; + +cfg_rt_multi_thread! { + use crate::runtime::Builder; + use crate::runtime::scheduler::MultiThread; +} + +/// The Tokio runtime. +/// +/// The runtime provides an I/O driver, task scheduler, [timer], and +/// blocking pool, necessary for running asynchronous tasks. +/// +/// Instances of `Runtime` can be created using [`new`], or [`Builder`]. +/// However, most users will use the `#[tokio::main]` annotation on their +/// entry point instead. +/// +/// See [module level][mod] documentation for more details. +/// +/// # Shutdown +/// +/// Shutting down the runtime is done by dropping the value. The current +/// thread will block until the shut down operation has completed. +/// +/// * Drain any scheduled work queues. +/// * Drop any futures that have not yet completed. +/// * Drop the reactor. +/// +/// Once the reactor has dropped, any outstanding I/O resources bound to +/// that reactor will no longer function. Calling any method on them will +/// result in an error. +/// +/// # Sharing +/// +/// The Tokio runtime implements `Sync` and `Send` to allow you to wrap it +/// in a `Arc`. Most fn take `&self` to allow you to call them concurrently +/// across multiple threads. +/// +/// Calls to `shutdown` and `shutdown_timeout` require exclusive ownership of +/// the runtime type and this can be achieved via `Arc::try_unwrap` when only +/// one strong count reference is left over. +/// +/// [timer]: crate::time +/// [mod]: index.html +/// [`new`]: method@Self::new +/// [`Builder`]: struct@Builder +#[derive(Debug)] +pub struct Runtime { + /// Task scheduler + scheduler: Scheduler, + + /// Handle to runtime, also contains driver handles + handle: Handle, + + /// Blocking pool handle, used to signal shutdown + blocking_pool: BlockingPool, +} + +/// The flavor of a `Runtime`. +/// +/// This is the return type for [`Handle::runtime_flavor`](crate::runtime::Handle::runtime_flavor()). +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum RuntimeFlavor { + /// The flavor that executes all tasks on the current thread. + CurrentThread, + /// The flavor that executes tasks across multiple threads. + MultiThread, +} + +/// The runtime scheduler is either a multi-thread or a current-thread executor. +#[derive(Debug)] +pub(super) enum Scheduler { + /// Execute all tasks on the current-thread. + CurrentThread(CurrentThread), + + /// Execute tasks across multiple threads. + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + MultiThread(MultiThread), +} + +impl Runtime { + pub(super) fn from_parts( + scheduler: Scheduler, + handle: Handle, + blocking_pool: BlockingPool, + ) -> Runtime { + Runtime { + scheduler, + handle, + blocking_pool, + } + } + + cfg_not_wasi! { + /// Creates a new runtime instance with default configuration values. + /// + /// This results in the multi threaded scheduler, I/O driver, and time driver being + /// initialized. + /// + /// Most applications will not need to call this function directly. Instead, + /// they will use the [`#[tokio::main]` attribute][main]. When a more complex + /// configuration is necessary, the [runtime builder] may be used. + /// + /// See [module level][mod] documentation for more details. + /// + /// # Examples + /// + /// Creating a new `Runtime` with default configuration values. + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// let rt = Runtime::new() + /// .unwrap(); + /// + /// // Use the runtime... + /// ``` + /// + /// [mod]: index.html + /// [main]: ../attr.main.html + /// [threaded scheduler]: index.html#threaded-scheduler + /// [runtime builder]: crate::runtime::Builder + #[cfg(feature = "rt-multi-thread")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] + pub fn new() -> std::io::Result<Runtime> { + Builder::new_multi_thread().enable_all().build() + } + } + + /// Returns a handle to the runtime's spawner. + /// + /// The returned handle can be used to spawn tasks that run on this runtime, and can + /// be cloned to allow moving the `Handle` to other threads. + /// + /// Calling [`Handle::block_on`] on a handle to a `current_thread` runtime is error-prone. + /// Refer to the documentation of [`Handle::block_on`] for more. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// let rt = Runtime::new() + /// .unwrap(); + /// + /// let handle = rt.handle(); + /// + /// // Use the handle... + /// ``` + pub fn handle(&self) -> &Handle { + &self.handle + } + + /// Spawns a future onto the Tokio runtime. + /// + /// This spawns the given future onto the runtime's executor, usually a + /// thread pool. The thread pool is then responsible for polling the future + /// until it completes. + /// + /// The provided future will start running in the background immediately + /// when `spawn` is called, even if you don't await the returned + /// `JoinHandle`. + /// + /// See [module level][mod] documentation for more details. + /// + /// [mod]: index.html + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// # fn dox() { + /// // Create the runtime + /// let rt = Runtime::new().unwrap(); + /// + /// // Spawn a future onto the runtime + /// rt.spawn(async { + /// println!("now running on a worker thread"); + /// }); + /// # } + /// ``` + #[track_caller] + pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output> + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + self.handle.spawn(future) + } + + /// Runs the provided function on an executor dedicated to blocking operations. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// # fn dox() { + /// // Create the runtime + /// let rt = Runtime::new().unwrap(); + /// + /// // Spawn a blocking function onto the runtime + /// rt.spawn_blocking(|| { + /// println!("now running on a worker thread"); + /// }); + /// # } + #[track_caller] + pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R> + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + self.handle.spawn_blocking(func) + } + + /// Runs a future to completion on the Tokio runtime. This is the + /// runtime's entry point. + /// + /// This runs the given future on the current thread, blocking until it is + /// complete, and yielding its resolved result. Any tasks or timers + /// which the future spawns internally will be executed on the runtime. + /// + /// # Multi thread scheduler + /// + /// When the multi thread scheduler is used this will allow futures + /// to run within the io driver and timer context of the overall runtime. + /// + /// Any spawned tasks will continue running after `block_on` returns. + /// + /// # Current thread scheduler + /// + /// When the current thread scheduler is enabled `block_on` + /// can be called concurrently from multiple threads. The first call + /// will take ownership of the io and timer drivers. This means + /// other threads which do not own the drivers will hook into that one. + /// When the first `block_on` completes, other threads will be able to + /// "steal" the driver to allow continued execution of their futures. + /// + /// Any spawned tasks will be suspended after `block_on` returns. Calling + /// `block_on` again will resume previously spawned tasks. + /// + /// # Panics + /// + /// This function panics if the provided future panics, or if called within an + /// asynchronous execution context. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::runtime::Runtime; + /// + /// // Create the runtime + /// let rt = Runtime::new().unwrap(); + /// + /// // Execute the future, blocking the current thread until completion + /// rt.block_on(async { + /// println!("hello"); + /// }); + /// ``` + /// + /// [handle]: fn@Handle::block_on + #[track_caller] + pub fn block_on<F: Future>(&self, future: F) -> F::Output { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let future = crate::util::trace::task( + future, + "block_on", + None, + crate::runtime::task::Id::next().as_u64(), + ); + + let _enter = self.enter(); + + match &self.scheduler { + Scheduler::CurrentThread(exec) => exec.block_on(&self.handle.inner, future), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Scheduler::MultiThread(exec) => exec.block_on(&self.handle.inner, future), + } + } + + /// Enters the runtime context. + /// + /// This allows you to construct types that must have an executor + /// available on creation such as [`Sleep`] or [`TcpStream`]. It will + /// also allow you to call methods such as [`tokio::spawn`]. + /// + /// [`Sleep`]: struct@crate::time::Sleep + /// [`TcpStream`]: struct@crate::net::TcpStream + /// [`tokio::spawn`]: fn@crate::spawn + /// + /// # Example + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// fn function_that_spawns(msg: String) { + /// // Had we not used `rt.enter` below, this would panic. + /// tokio::spawn(async move { + /// println!("{}", msg); + /// }); + /// } + /// + /// fn main() { + /// let rt = Runtime::new().unwrap(); + /// + /// let s = "Hello World!".to_string(); + /// + /// // By entering the context, we tie `tokio::spawn` to this executor. + /// let _guard = rt.enter(); + /// function_that_spawns(s); + /// } + /// ``` + pub fn enter(&self) -> EnterGuard<'_> { + self.handle.enter() + } + + /// Shuts down the runtime, waiting for at most `duration` for all spawned + /// task to shutdown. + /// + /// Usually, dropping a `Runtime` handle is sufficient as tasks are able to + /// shutdown in a timely fashion. However, dropping a `Runtime` will wait + /// indefinitely for all tasks to terminate, and there are cases where a long + /// blocking task has been spawned, which can block dropping `Runtime`. + /// + /// In this case, calling `shutdown_timeout` with an explicit wait timeout + /// can work. The `shutdown_timeout` will signal all tasks to shutdown and + /// will wait for at most `duration` for all spawned tasks to terminate. If + /// `timeout` elapses before all tasks are dropped, the function returns and + /// outstanding tasks are potentially leaked. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// use tokio::task; + /// + /// use std::thread; + /// use std::time::Duration; + /// + /// fn main() { + /// let runtime = Runtime::new().unwrap(); + /// + /// runtime.block_on(async move { + /// task::spawn_blocking(move || { + /// thread::sleep(Duration::from_secs(10_000)); + /// }); + /// }); + /// + /// runtime.shutdown_timeout(Duration::from_millis(100)); + /// } + /// ``` + pub fn shutdown_timeout(mut self, duration: Duration) { + // Wakeup and shutdown all the worker threads + self.handle.inner.shutdown(); + self.blocking_pool.shutdown(Some(duration)); + } + + /// Shuts down the runtime, without waiting for any spawned tasks to shutdown. + /// + /// This can be useful if you want to drop a runtime from within another runtime. + /// Normally, dropping a runtime will block indefinitely for spawned blocking tasks + /// to complete, which would normally not be permitted within an asynchronous context. + /// By calling `shutdown_background()`, you can drop the runtime from such a context. + /// + /// Note however, that because we do not wait for any blocking tasks to complete, this + /// may result in a resource leak (in that any blocking tasks are still running until they + /// return. + /// + /// This function is equivalent to calling `shutdown_timeout(Duration::from_nanos(0))`. + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// fn main() { + /// let runtime = Runtime::new().unwrap(); + /// + /// runtime.block_on(async move { + /// let inner_runtime = Runtime::new().unwrap(); + /// // ... + /// inner_runtime.shutdown_background(); + /// }); + /// } + /// ``` + pub fn shutdown_background(self) { + self.shutdown_timeout(Duration::from_nanos(0)) + } +} + +#[allow(clippy::single_match)] // there are comments in the error branch, so we don't want if-let +impl Drop for Runtime { + fn drop(&mut self) { + match &mut self.scheduler { + Scheduler::CurrentThread(current_thread) => { + // This ensures that tasks spawned on the current-thread + // runtime are dropped inside the runtime's context. + let _guard = context::try_set_current(&self.handle.inner); + current_thread.shutdown(&self.handle.inner); + } + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Scheduler::MultiThread(multi_thread) => { + // The threaded scheduler drops its tasks on its worker threads, which is + // already in the runtime's context. + multi_thread.shutdown(&self.handle.inner); + } + } + } +} + +cfg_metrics! { + impl Runtime { + /// TODO + pub fn metrics(&self) -> crate::runtime::RuntimeMetrics { + self.handle.metrics() + } + } +} diff --git a/src/runtime/scheduler/current_thread.rs b/src/runtime/scheduler/current_thread.rs new file mode 100644 index 0000000..375e47c --- /dev/null +++ b/src/runtime/scheduler/current_thread.rs @@ -0,0 +1,634 @@ +use crate::future::poll_fn; +use crate::loom::sync::atomic::AtomicBool; +use crate::loom::sync::{Arc, Mutex}; +use crate::runtime::driver::{self, Driver}; +use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task}; +use crate::runtime::{blocking, context, scheduler, Config}; +use crate::runtime::{MetricsBatch, SchedulerMetrics, WorkerMetrics}; +use crate::sync::notify::Notify; +use crate::util::atomic_cell::AtomicCell; +use crate::util::{waker_ref, RngSeedGenerator, Wake, WakerRef}; + +use std::cell::RefCell; +use std::collections::VecDeque; +use std::fmt; +use std::future::Future; +use std::sync::atomic::Ordering::{AcqRel, Release}; +use std::task::Poll::{Pending, Ready}; +use std::time::Duration; + +/// Executes tasks on the current thread +pub(crate) struct CurrentThread { + /// Core scheduler data is acquired by a thread entering `block_on`. + core: AtomicCell<Core>, + + /// Notifier for waking up other threads to steal the + /// driver. + notify: Notify, +} + +/// Handle to the current thread scheduler +pub(crate) struct Handle { + /// Scheduler state shared across threads + shared: Shared, + + /// Resource driver handles + pub(crate) driver: driver::Handle, + + /// Blocking pool spawner + pub(crate) blocking_spawner: blocking::Spawner, + + /// Current random number generator seed + pub(crate) seed_generator: RngSeedGenerator, +} + +/// Data required for executing the scheduler. The struct is passed around to +/// a function that will perform the scheduling work and acts as a capability token. +struct Core { + /// Scheduler run queue + tasks: VecDeque<task::Notified<Arc<Handle>>>, + + /// Current tick + tick: u32, + + /// Runtime driver + /// + /// The driver is removed before starting to park the thread + driver: Option<Driver>, + + /// Metrics batch + metrics: MetricsBatch, + + /// True if a task panicked without being handled and the runtime is + /// configured to shutdown on unhandled panic. + unhandled_panic: bool, +} + +/// Scheduler state shared between threads. +struct Shared { + /// Remote run queue. None if the `Runtime` has been dropped. + queue: Mutex<Option<VecDeque<task::Notified<Arc<Handle>>>>>, + + /// Collection of all active tasks spawned onto this executor. + owned: OwnedTasks<Arc<Handle>>, + + /// Indicates whether the blocked on thread was woken. + woken: AtomicBool, + + /// Scheduler configuration options + config: Config, + + /// Keeps track of various runtime metrics. + scheduler_metrics: SchedulerMetrics, + + /// This scheduler only has one worker. + worker_metrics: WorkerMetrics, +} + +/// Thread-local context. +struct Context { + /// Scheduler handle + handle: Arc<Handle>, + + /// Scheduler core, enabling the holder of `Context` to execute the + /// scheduler. + core: RefCell<Option<Box<Core>>>, +} + +/// Initial queue capacity. +const INITIAL_CAPACITY: usize = 64; + +// Tracks the current CurrentThread. +scoped_thread_local!(static CURRENT: Context); + +impl CurrentThread { + pub(crate) fn new( + driver: Driver, + driver_handle: driver::Handle, + blocking_spawner: blocking::Spawner, + seed_generator: RngSeedGenerator, + config: Config, + ) -> (CurrentThread, Arc<Handle>) { + let handle = Arc::new(Handle { + shared: Shared { + queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))), + owned: OwnedTasks::new(), + woken: AtomicBool::new(false), + config, + scheduler_metrics: SchedulerMetrics::new(), + worker_metrics: WorkerMetrics::new(), + }, + driver: driver_handle, + blocking_spawner, + seed_generator, + }); + + let core = AtomicCell::new(Some(Box::new(Core { + tasks: VecDeque::with_capacity(INITIAL_CAPACITY), + tick: 0, + driver: Some(driver), + metrics: MetricsBatch::new(), + unhandled_panic: false, + }))); + + let scheduler = CurrentThread { + core, + notify: Notify::new(), + }; + + (scheduler, handle) + } + + #[track_caller] + pub(crate) fn block_on<F: Future>(&self, handle: &scheduler::Handle, future: F) -> F::Output { + pin!(future); + + let mut enter = crate::runtime::context::enter_runtime(handle, false); + let handle = handle.as_current_thread(); + + // Attempt to steal the scheduler core and block_on the future if we can + // there, otherwise, lets select on a notification that the core is + // available or the future is complete. + loop { + if let Some(core) = self.take_core(handle) { + return core.block_on(future); + } else { + let notified = self.notify.notified(); + pin!(notified); + + if let Some(out) = enter + .blocking + .block_on(poll_fn(|cx| { + if notified.as_mut().poll(cx).is_ready() { + return Ready(None); + } + + if let Ready(out) = future.as_mut().poll(cx) { + return Ready(Some(out)); + } + + Pending + })) + .expect("Failed to `Enter::block_on`") + { + return out; + } + } + } + } + + fn take_core(&self, handle: &Arc<Handle>) -> Option<CoreGuard<'_>> { + let core = self.core.take()?; + + Some(CoreGuard { + context: Context { + handle: handle.clone(), + core: RefCell::new(Some(core)), + }, + scheduler: self, + }) + } + + pub(crate) fn shutdown(&mut self, handle: &scheduler::Handle) { + let handle = handle.as_current_thread(); + + // Avoid a double panic if we are currently panicking and + // the lock may be poisoned. + + let core = match self.take_core(handle) { + Some(core) => core, + None if std::thread::panicking() => return, + None => panic!("Oh no! We never placed the Core back, this is a bug!"), + }; + + core.enter(|mut core, _context| { + // Drain the OwnedTasks collection. This call also closes the + // collection, ensuring that no tasks are ever pushed after this + // call returns. + handle.shared.owned.close_and_shutdown_all(); + + // Drain local queue + // We already shut down every task, so we just need to drop the task. + while let Some(task) = core.pop_task(handle) { + drop(task); + } + + // Drain remote queue and set it to None + let remote_queue = handle.shared.queue.lock().take(); + + // Using `Option::take` to replace the shared queue with `None`. + // We already shut down every task, so we just need to drop the task. + if let Some(remote_queue) = remote_queue { + for task in remote_queue { + drop(task); + } + } + + assert!(handle.shared.owned.is_empty()); + + // Submit metrics + core.metrics.submit(&handle.shared.worker_metrics); + + // Shutdown the resource drivers + if let Some(driver) = core.driver.as_mut() { + driver.shutdown(&handle.driver); + } + + (core, ()) + }); + } +} + +impl fmt::Debug for CurrentThread { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("CurrentThread").finish() + } +} + +// ===== impl Core ===== + +impl Core { + fn pop_task(&mut self, handle: &Handle) -> Option<task::Notified<Arc<Handle>>> { + let ret = self.tasks.pop_front(); + handle + .shared + .worker_metrics + .set_queue_depth(self.tasks.len()); + ret + } + + fn push_task(&mut self, handle: &Handle, task: task::Notified<Arc<Handle>>) { + self.tasks.push_back(task); + self.metrics.inc_local_schedule_count(); + handle + .shared + .worker_metrics + .set_queue_depth(self.tasks.len()); + } +} + +fn did_defer_tasks() -> bool { + context::with_defer(|deferred| !deferred.is_empty()).unwrap() +} + +fn wake_deferred_tasks() { + context::with_defer(|deferred| deferred.wake()); +} + +// ===== impl Context ===== + +impl Context { + /// Execute the closure with the given scheduler core stored in the + /// thread-local context. + fn run_task<R>(&self, mut core: Box<Core>, f: impl FnOnce() -> R) -> (Box<Core>, R) { + core.metrics.incr_poll_count(); + self.enter(core, || crate::runtime::coop::budget(f)) + } + + /// Blocks the current thread until an event is received by the driver, + /// including I/O events, timer events, ... + fn park(&self, mut core: Box<Core>, handle: &Handle) -> Box<Core> { + let mut driver = core.driver.take().expect("driver missing"); + + if let Some(f) = &handle.shared.config.before_park { + // Incorrect lint, the closures are actually different types so `f` + // cannot be passed as an argument to `enter`. + #[allow(clippy::redundant_closure)] + let (c, _) = self.enter(core, || f()); + core = c; + } + + // This check will fail if `before_park` spawns a task for us to run + // instead of parking the thread + if core.tasks.is_empty() { + // Park until the thread is signaled + core.metrics.about_to_park(); + core.metrics.submit(&handle.shared.worker_metrics); + + let (c, _) = self.enter(core, || { + driver.park(&handle.driver); + wake_deferred_tasks(); + }); + + core = c; + core.metrics.returned_from_park(); + } + + if let Some(f) = &handle.shared.config.after_unpark { + // Incorrect lint, the closures are actually different types so `f` + // cannot be passed as an argument to `enter`. + #[allow(clippy::redundant_closure)] + let (c, _) = self.enter(core, || f()); + core = c; + } + + core.driver = Some(driver); + core + } + + /// Checks the driver for new events without blocking the thread. + fn park_yield(&self, mut core: Box<Core>, handle: &Handle) -> Box<Core> { + let mut driver = core.driver.take().expect("driver missing"); + + core.metrics.submit(&handle.shared.worker_metrics); + let (mut core, _) = self.enter(core, || { + driver.park_timeout(&handle.driver, Duration::from_millis(0)); + wake_deferred_tasks(); + }); + + core.driver = Some(driver); + core + } + + fn enter<R>(&self, core: Box<Core>, f: impl FnOnce() -> R) -> (Box<Core>, R) { + // Store the scheduler core in the thread-local context + // + // A drop-guard is employed at a higher level. + *self.core.borrow_mut() = Some(core); + + // Execute the closure while tracking the execution budget + let ret = f(); + + // Take the scheduler core back + let core = self.core.borrow_mut().take().expect("core missing"); + (core, ret) + } +} + +// ===== impl Handle ===== + +impl Handle { + /// Spawns a future onto the `CurrentThread` scheduler + pub(crate) fn spawn<F>( + me: &Arc<Self>, + future: F, + id: crate::runtime::task::Id, + ) -> JoinHandle<F::Output> + where + F: crate::future::Future + Send + 'static, + F::Output: Send + 'static, + { + let (handle, notified) = me.shared.owned.bind(future, me.clone(), id); + + if let Some(notified) = notified { + me.schedule(notified); + } + + handle + } + + fn pop(&self) -> Option<task::Notified<Arc<Handle>>> { + match self.shared.queue.lock().as_mut() { + Some(queue) => queue.pop_front(), + None => None, + } + } + + fn waker_ref(me: &Arc<Self>) -> WakerRef<'_> { + // Set woken to true when enter block_on, ensure outer future + // be polled for the first time when enter loop + me.shared.woken.store(true, Release); + waker_ref(me) + } + + // reset woken to false and return original value + pub(crate) fn reset_woken(&self) -> bool { + self.shared.woken.swap(false, AcqRel) + } +} + +cfg_metrics! { + impl Handle { + pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { + &self.shared.scheduler_metrics + } + + pub(crate) fn injection_queue_depth(&self) -> usize { + // TODO: avoid having to lock. The multi-threaded injection queue + // could probably be used here. + self.shared + .queue + .lock() + .as_ref() + .map(|queue| queue.len()) + .unwrap_or(0) + } + + pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { + assert_eq!(0, worker); + &self.shared.worker_metrics + } + + pub(crate) fn num_blocking_threads(&self) -> usize { + self.blocking_spawner.num_threads() + } + + pub(crate) fn num_idle_blocking_threads(&self) -> usize { + self.blocking_spawner.num_idle_threads() + } + + pub(crate) fn blocking_queue_depth(&self) -> usize { + self.blocking_spawner.queue_depth() + } + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("current_thread::Handle { ... }").finish() + } +} + +// ===== impl Shared ===== + +impl Schedule for Arc<Handle> { + fn release(&self, task: &Task<Self>) -> Option<Task<Self>> { + self.shared.owned.remove(task) + } + + fn schedule(&self, task: task::Notified<Self>) { + CURRENT.with(|maybe_cx| match maybe_cx { + Some(cx) if Arc::ptr_eq(self, &cx.handle) => { + let mut core = cx.core.borrow_mut(); + + // If `None`, the runtime is shutting down, so there is no need + // to schedule the task. + if let Some(core) = core.as_mut() { + core.push_task(self, task); + } + } + _ => { + // Track that a task was scheduled from **outside** of the runtime. + self.shared.scheduler_metrics.inc_remote_schedule_count(); + + // If the queue is None, then the runtime has shut down. We + // don't need to do anything with the notification in that case. + let mut guard = self.shared.queue.lock(); + if let Some(queue) = guard.as_mut() { + queue.push_back(task); + drop(guard); + self.driver.unpark(); + } + } + }); + } + + cfg_unstable! { + fn unhandled_panic(&self) { + use crate::runtime::UnhandledPanic; + + match self.shared.config.unhandled_panic { + UnhandledPanic::Ignore => { + // Do nothing + } + UnhandledPanic::ShutdownRuntime => { + // This hook is only called from within the runtime, so + // `CURRENT` should match with `&self`, i.e. there is no + // opportunity for a nested scheduler to be called. + CURRENT.with(|maybe_cx| match maybe_cx { + Some(cx) if Arc::ptr_eq(self, &cx.handle) => { + let mut core = cx.core.borrow_mut(); + + // If `None`, the runtime is shutting down, so there is no need to signal shutdown + if let Some(core) = core.as_mut() { + core.unhandled_panic = true; + self.shared.owned.close_and_shutdown_all(); + } + } + _ => unreachable!("runtime core not set in CURRENT thread-local"), + }) + } + } + } + } +} + +impl Wake for Handle { + fn wake(arc_self: Arc<Self>) { + Wake::wake_by_ref(&arc_self) + } + + /// Wake by reference + fn wake_by_ref(arc_self: &Arc<Self>) { + arc_self.shared.woken.store(true, Release); + arc_self.driver.unpark(); + } +} + +// ===== CoreGuard ===== + +/// Used to ensure we always place the `Core` value back into its slot in +/// `CurrentThread`, even if the future panics. +struct CoreGuard<'a> { + context: Context, + scheduler: &'a CurrentThread, +} + +impl CoreGuard<'_> { + #[track_caller] + fn block_on<F: Future>(self, future: F) -> F::Output { + let ret = self.enter(|mut core, context| { + let waker = Handle::waker_ref(&context.handle); + let mut cx = std::task::Context::from_waker(&waker); + + pin!(future); + + 'outer: loop { + let handle = &context.handle; + + if handle.reset_woken() { + let (c, res) = context.enter(core, || { + crate::runtime::coop::budget(|| future.as_mut().poll(&mut cx)) + }); + + core = c; + + if let Ready(v) = res { + return (core, Some(v)); + } + } + + for _ in 0..handle.shared.config.event_interval { + // Make sure we didn't hit an unhandled_panic + if core.unhandled_panic { + return (core, None); + } + + // Get and increment the current tick + let tick = core.tick; + core.tick = core.tick.wrapping_add(1); + + let entry = if tick % handle.shared.config.global_queue_interval == 0 { + handle.pop().or_else(|| core.tasks.pop_front()) + } else { + core.tasks.pop_front().or_else(|| handle.pop()) + }; + + let task = match entry { + Some(entry) => entry, + None => { + core = if did_defer_tasks() { + context.park_yield(core, handle) + } else { + context.park(core, handle) + }; + + // Try polling the `block_on` future next + continue 'outer; + } + }; + + let task = context.handle.shared.owned.assert_owner(task); + + let (c, _) = context.run_task(core, || { + task.run(); + }); + + core = c; + } + + // Yield to the driver, this drives the timer and pulls any + // pending I/O events. + core = context.park_yield(core, handle); + } + }); + + match ret { + Some(ret) => ret, + None => { + // `block_on` panicked. + panic!("a spawned task panicked and the runtime is configured to shut down on unhandled panic"); + } + } + } + + /// Enters the scheduler context. This sets the queue and other necessary + /// scheduler state in the thread-local. + fn enter<F, R>(self, f: F) -> R + where + F: FnOnce(Box<Core>, &Context) -> (Box<Core>, R), + { + // Remove `core` from `context` to pass into the closure. + let core = self.context.core.borrow_mut().take().expect("core missing"); + + // Call the closure and place `core` back + let (core, ret) = CURRENT.set(&self.context, || f(core, &self.context)); + + *self.context.core.borrow_mut() = Some(core); + + ret + } +} + +impl Drop for CoreGuard<'_> { + fn drop(&mut self) { + if let Some(core) = self.context.core.borrow_mut().take() { + // Replace old scheduler back into the state to allow + // other threads to pick it up and drive it. + self.scheduler.core.set(core); + + // Wake up other possible threads that could steal the driver. + self.scheduler.notify.notify_one() + } + } +} diff --git a/src/runtime/scheduler/mod.rs b/src/runtime/scheduler/mod.rs new file mode 100644 index 0000000..f45d8a8 --- /dev/null +++ b/src/runtime/scheduler/mod.rs @@ -0,0 +1,194 @@ +cfg_rt! { + pub(crate) mod current_thread; + pub(crate) use current_thread::CurrentThread; +} + +cfg_rt_multi_thread! { + pub(crate) mod multi_thread; + pub(crate) use multi_thread::MultiThread; +} + +use crate::runtime::driver; + +#[derive(Debug, Clone)] +pub(crate) enum Handle { + #[cfg(feature = "rt")] + CurrentThread(Arc<current_thread::Handle>), + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + MultiThread(Arc<multi_thread::Handle>), + + // TODO: This is to avoid triggering "dead code" warnings many other places + // in the codebase. Remove this during a later cleanup + #[cfg(not(feature = "rt"))] + #[allow(dead_code)] + Disabled, +} + +impl Handle { + #[cfg_attr(not(feature = "full"), allow(dead_code))] + pub(crate) fn driver(&self) -> &driver::Handle { + match *self { + #[cfg(feature = "rt")] + Handle::CurrentThread(ref h) => &h.driver, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(ref h) => &h.driver, + + #[cfg(not(feature = "rt"))] + Handle::Disabled => unreachable!(), + } + } +} + +cfg_rt! { + use crate::future::Future; + use crate::loom::sync::Arc; + use crate::runtime::{blocking, task::Id}; + use crate::runtime::context; + use crate::task::JoinHandle; + use crate::util::RngSeedGenerator; + + impl Handle { + #[track_caller] + pub(crate) fn current() -> Handle { + match context::try_current() { + Ok(handle) => handle, + Err(e) => panic!("{}", e), + } + } + + pub(crate) fn blocking_spawner(&self) -> &blocking::Spawner { + match self { + Handle::CurrentThread(h) => &h.blocking_spawner, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(h) => &h.blocking_spawner, + } + } + + pub(crate) fn spawn<F>(&self, future: F, id: Id) -> JoinHandle<F::Output> + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + match self { + Handle::CurrentThread(h) => current_thread::Handle::spawn(h, future, id), + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(h) => multi_thread::Handle::spawn(h, future, id), + } + } + + pub(crate) fn shutdown(&self) { + match *self { + Handle::CurrentThread(_) => {}, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(ref h) => h.shutdown(), + } + } + + pub(crate) fn seed_generator(&self) -> &RngSeedGenerator { + match self { + Handle::CurrentThread(h) => &h.seed_generator, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(h) => &h.seed_generator, + } + } + + pub(crate) fn as_current_thread(&self) -> &Arc<current_thread::Handle> { + match self { + Handle::CurrentThread(handle) => handle, + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + _ => panic!("not a CurrentThread handle"), + } + } + } + + cfg_metrics! { + use crate::runtime::{SchedulerMetrics, WorkerMetrics}; + + impl Handle { + pub(crate) fn num_workers(&self) -> usize { + match self { + Handle::CurrentThread(_) => 1, + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.num_workers(), + } + } + + pub(crate) fn num_blocking_threads(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.num_blocking_threads(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.num_blocking_threads(), + } + } + + pub(crate) fn num_idle_blocking_threads(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.num_idle_blocking_threads(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.num_idle_blocking_threads(), + } + } + + pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { + match self { + Handle::CurrentThread(handle) => handle.scheduler_metrics(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.scheduler_metrics(), + } + } + + pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { + match self { + Handle::CurrentThread(handle) => handle.worker_metrics(worker), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.worker_metrics(worker), + } + } + + pub(crate) fn injection_queue_depth(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.injection_queue_depth(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.injection_queue_depth(), + } + } + + pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize { + match self { + Handle::CurrentThread(handle) => handle.worker_metrics(worker).queue_depth(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.worker_local_queue_depth(worker), + } + } + + pub(crate) fn blocking_queue_depth(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.blocking_queue_depth(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.blocking_queue_depth(), + } + } + } + } +} + +cfg_not_rt! { + #[cfg(any( + feature = "net", + all(unix, feature = "process"), + all(unix, feature = "signal"), + feature = "time", + ))] + impl Handle { + #[track_caller] + pub(crate) fn current() -> Handle { + panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) + } + } +} diff --git a/src/runtime/scheduler/multi_thread/handle.rs b/src/runtime/scheduler/multi_thread/handle.rs new file mode 100644 index 0000000..69a4620 --- /dev/null +++ b/src/runtime/scheduler/multi_thread/handle.rs @@ -0,0 +1,98 @@ +use crate::future::Future; +use crate::loom::sync::Arc; +use crate::runtime::scheduler::multi_thread::worker; +use crate::runtime::{ + blocking, driver, + task::{self, JoinHandle}, +}; +use crate::util::RngSeedGenerator; + +use std::fmt; + +/// Handle to the multi thread scheduler +pub(crate) struct Handle { + /// Task spawner + pub(super) shared: worker::Shared, + + /// Resource driver handles + pub(crate) driver: driver::Handle, + + /// Blocking pool spawner + pub(crate) blocking_spawner: blocking::Spawner, + + /// Current random number generator seed + pub(crate) seed_generator: RngSeedGenerator, +} + +impl Handle { + /// Spawns a future onto the thread pool + pub(crate) fn spawn<F>(me: &Arc<Self>, future: F, id: task::Id) -> JoinHandle<F::Output> + where + F: crate::future::Future + Send + 'static, + F::Output: Send + 'static, + { + Self::bind_new_task(me, future, id) + } + + pub(crate) fn shutdown(&self) { + self.close(); + } + + pub(super) fn bind_new_task<T>(me: &Arc<Self>, future: T, id: task::Id) -> JoinHandle<T::Output> + where + T: Future + Send + 'static, + T::Output: Send + 'static, + { + let (handle, notified) = me.shared.owned.bind(future, me.clone(), id); + + if let Some(notified) = notified { + me.schedule_task(notified, false); + } + + handle + } +} + +cfg_metrics! { + use crate::runtime::{SchedulerMetrics, WorkerMetrics}; + + impl Handle { + pub(crate) fn num_workers(&self) -> usize { + self.shared.worker_metrics.len() + } + + pub(crate) fn num_blocking_threads(&self) -> usize { + self.blocking_spawner.num_threads() + } + + pub(crate) fn num_idle_blocking_threads(&self) -> usize { + self.blocking_spawner.num_idle_threads() + } + + pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { + &self.shared.scheduler_metrics + } + + pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { + &self.shared.worker_metrics[worker] + } + + pub(crate) fn injection_queue_depth(&self) -> usize { + self.shared.injection_queue_depth() + } + + pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize { + self.shared.worker_local_queue_depth(worker) + } + + pub(crate) fn blocking_queue_depth(&self) -> usize { + self.blocking_spawner.queue_depth() + } + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("multi_thread::Handle { ... }").finish() + } +} diff --git a/src/runtime/thread_pool/idle.rs b/src/runtime/scheduler/multi_thread/idle.rs index 6b7ee12..a57bf6a 100644 --- a/src/runtime/thread_pool/idle.rs +++ b/src/runtime/scheduler/multi_thread/idle.rs @@ -64,7 +64,7 @@ impl Idle { // A worker should be woken up, atomically increment the number of // searching workers as well as the number of unparked workers. - State::unpark_one(&self.state); + State::unpark_one(&self.state, 1); // Get the worker to unpark let ret = sleepers.pop(); @@ -111,7 +111,9 @@ impl Idle { /// Unpark a specific worker. This happens if tasks are submitted from /// within the worker's park routine. - pub(super) fn unpark_worker_by_id(&self, worker_id: usize) { + /// + /// Returns `true` if the worker was parked before calling the method. + pub(super) fn unpark_worker_by_id(&self, worker_id: usize) -> bool { let mut sleepers = self.sleepers.lock(); for index in 0..sleepers.len() { @@ -119,11 +121,13 @@ impl Idle { sleepers.swap_remove(index); // Update the state accordingly while the lock is held. - State::unpark_one(&self.state); + State::unpark_one(&self.state, 0); - return; + return true; } } + + false } /// Returns `true` if `worker_id` is contained in the sleep set. @@ -151,8 +155,8 @@ impl State { State(cell.load(ordering)) } - fn unpark_one(cell: &AtomicUsize) { - cell.fetch_add(1 | (1 << UNPARK_SHIFT), SeqCst); + fn unpark_one(cell: &AtomicUsize, num_searching: usize) { + cell.fetch_add(num_searching | (1 << UNPARK_SHIFT), SeqCst); } fn inc_num_searching(cell: &AtomicUsize, ordering: Ordering) { diff --git a/src/runtime/scheduler/multi_thread/mod.rs b/src/runtime/scheduler/multi_thread/mod.rs new file mode 100644 index 0000000..47cd1f3 --- /dev/null +++ b/src/runtime/scheduler/multi_thread/mod.rs @@ -0,0 +1,84 @@ +//! Multi-threaded runtime + +mod handle; +pub(crate) use handle::Handle; + +mod idle; +use self::idle::Idle; + +mod park; +pub(crate) use park::{Parker, Unparker}; + +pub(crate) mod queue; + +mod worker; +pub(crate) use worker::Launch; + +pub(crate) use worker::block_in_place; + +use crate::loom::sync::Arc; +use crate::runtime::{ + blocking, + driver::{self, Driver}, + scheduler, Config, +}; +use crate::util::RngSeedGenerator; + +use std::fmt; +use std::future::Future; + +/// Work-stealing based thread pool for executing futures. +pub(crate) struct MultiThread; + +// ===== impl MultiThread ===== + +impl MultiThread { + pub(crate) fn new( + size: usize, + driver: Driver, + driver_handle: driver::Handle, + blocking_spawner: blocking::Spawner, + seed_generator: RngSeedGenerator, + config: Config, + ) -> (MultiThread, Arc<Handle>, Launch) { + let parker = Parker::new(driver); + let (handle, launch) = worker::create( + size, + parker, + driver_handle, + blocking_spawner, + seed_generator, + config, + ); + + (MultiThread, handle, launch) + } + + /// Blocks the current thread waiting for the future to complete. + /// + /// The future will execute on the current thread, but all spawned tasks + /// will be executed on the thread pool. + pub(crate) fn block_on<F>(&self, handle: &scheduler::Handle, future: F) -> F::Output + where + F: Future, + { + let mut enter = crate::runtime::context::enter_runtime(handle, true); + enter + .blocking + .block_on(future) + .expect("failed to park thread") + } + + pub(crate) fn shutdown(&mut self, handle: &scheduler::Handle) { + match handle { + scheduler::Handle::MultiThread(handle) => handle.shutdown(), + _ => panic!("expected MultiThread scheduler"), + } + } +} + +impl fmt::Debug for MultiThread { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("MultiThread").finish() + } +} diff --git a/src/runtime/scheduler/multi_thread/park.rs b/src/runtime/scheduler/multi_thread/park.rs new file mode 100644 index 0000000..6bdbff9 --- /dev/null +++ b/src/runtime/scheduler/multi_thread/park.rs @@ -0,0 +1,237 @@ +//! Parks the runtime. +//! +//! A combination of the various resource driver park handles. + +use crate::loom::sync::atomic::AtomicUsize; +use crate::loom::sync::{Arc, Condvar, Mutex}; +use crate::loom::thread; +use crate::runtime::driver::{self, Driver}; +use crate::util::TryLock; + +use std::sync::atomic::Ordering::SeqCst; +use std::time::Duration; + +pub(crate) struct Parker { + inner: Arc<Inner>, +} + +pub(crate) struct Unparker { + inner: Arc<Inner>, +} + +struct Inner { + /// Avoids entering the park if possible + state: AtomicUsize, + + /// Used to coordinate access to the driver / condvar + mutex: Mutex<()>, + + /// Condvar to block on if the driver is unavailable. + condvar: Condvar, + + /// Resource (I/O, time, ...) driver + shared: Arc<Shared>, +} + +const EMPTY: usize = 0; +const PARKED_CONDVAR: usize = 1; +const PARKED_DRIVER: usize = 2; +const NOTIFIED: usize = 3; + +/// Shared across multiple Parker handles +struct Shared { + /// Shared driver. Only one thread at a time can use this + driver: TryLock<Driver>, +} + +impl Parker { + pub(crate) fn new(driver: Driver) -> Parker { + Parker { + inner: Arc::new(Inner { + state: AtomicUsize::new(EMPTY), + mutex: Mutex::new(()), + condvar: Condvar::new(), + shared: Arc::new(Shared { + driver: TryLock::new(driver), + }), + }), + } + } + + pub(crate) fn unpark(&self) -> Unparker { + Unparker { + inner: self.inner.clone(), + } + } + + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.inner.park(handle); + } + + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + // Only parking with zero is supported... + assert_eq!(duration, Duration::from_millis(0)); + + if let Some(mut driver) = self.inner.shared.driver.try_lock() { + driver.park_timeout(handle, duration) + } + } + + pub(crate) fn shutdown(&mut self, handle: &driver::Handle) { + self.inner.shutdown(handle); + } +} + +impl Clone for Parker { + fn clone(&self) -> Parker { + Parker { + inner: Arc::new(Inner { + state: AtomicUsize::new(EMPTY), + mutex: Mutex::new(()), + condvar: Condvar::new(), + shared: self.inner.shared.clone(), + }), + } + } +} + +impl Unparker { + pub(crate) fn unpark(&self, driver: &driver::Handle) { + self.inner.unpark(driver); + } +} + +impl Inner { + /// Parks the current thread for at most `dur`. + fn park(&self, handle: &driver::Handle) { + for _ in 0..3 { + // If we were previously notified then we consume this notification and + // return quickly. + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + return; + } + + thread::yield_now(); + } + + if let Some(mut driver) = self.shared.driver.try_lock() { + self.park_driver(&mut driver, handle); + } else { + self.park_condvar(); + } + } + + fn park_condvar(&self) { + // Otherwise we need to coordinate going to sleep + let mut m = self.mutex.lock(); + + match self + .state + .compare_exchange(EMPTY, PARKED_CONDVAR, SeqCst, SeqCst) + { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read here, even though we know it will be `NOTIFIED`. + // This is because `unpark` may have been called again since we read + // `NOTIFIED` in the `compare_exchange` above. We must perform an + // acquire operation that synchronizes with that `unpark` to observe + // any writes it made before the call to unpark. To do that we must + // read from the write it made to `state`. + let old = self.state.swap(EMPTY, SeqCst); + debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + + return; + } + Err(actual) => panic!("inconsistent park state; actual = {}", actual), + } + + loop { + m = self.condvar.wait(m).unwrap(); + + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + // got a notification + return; + } + + // spurious wakeup, go back to sleep + } + } + + fn park_driver(&self, driver: &mut Driver, handle: &driver::Handle) { + match self + .state + .compare_exchange(EMPTY, PARKED_DRIVER, SeqCst, SeqCst) + { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read here, even though we know it will be `NOTIFIED`. + // This is because `unpark` may have been called again since we read + // `NOTIFIED` in the `compare_exchange` above. We must perform an + // acquire operation that synchronizes with that `unpark` to observe + // any writes it made before the call to unpark. To do that we must + // read from the write it made to `state`. + let old = self.state.swap(EMPTY, SeqCst); + debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + + return; + } + Err(actual) => panic!("inconsistent park state; actual = {}", actual), + } + + driver.park(handle); + + match self.state.swap(EMPTY, SeqCst) { + NOTIFIED => {} // got a notification, hurray! + PARKED_DRIVER => {} // no notification, alas + n => panic!("inconsistent park_timeout state: {}", n), + } + } + + fn unpark(&self, driver: &driver::Handle) { + // To ensure the unparked thread will observe any writes we made before + // this call, we must perform a release operation that `park` can + // synchronize with. To do that we must write `NOTIFIED` even if `state` + // is already `NOTIFIED`. That is why this must be a swap rather than a + // compare-and-swap that returns if it reads `NOTIFIED` on failure. + match self.state.swap(NOTIFIED, SeqCst) { + EMPTY => {} // no one was waiting + NOTIFIED => {} // already unparked + PARKED_CONDVAR => self.unpark_condvar(), + PARKED_DRIVER => driver.unpark(), + actual => panic!("inconsistent state in unpark; actual = {}", actual), + } + } + + fn unpark_condvar(&self) { + // There is a period between when the parked thread sets `state` to + // `PARKED` (or last checked `state` in the case of a spurious wake + // up) and when it actually waits on `cvar`. If we were to notify + // during this period it would be ignored and then when the parked + // thread went to sleep it would never wake up. Fortunately, it has + // `lock` locked at this stage so we can acquire `lock` to wait until + // it is ready to receive the notification. + // + // Releasing `lock` before the call to `notify_one` means that when the + // parked thread wakes it doesn't get woken only to have to wait for us + // to release `lock`. + drop(self.mutex.lock()); + + self.condvar.notify_one() + } + + fn shutdown(&self, handle: &driver::Handle) { + if let Some(mut driver) = self.shared.driver.try_lock() { + driver.shutdown(handle); + } + + self.condvar.notify_all(); + } +} diff --git a/src/runtime/queue.rs b/src/runtime/scheduler/multi_thread/queue.rs index a88dffc..faf56db 100644 --- a/src/runtime/queue.rs +++ b/src/runtime/scheduler/multi_thread/queue.rs @@ -1,39 +1,56 @@ //! Run-queue structures to support a work-stealing scheduler use crate::loom::cell::UnsafeCell; -use crate::loom::sync::atomic::{AtomicU16, AtomicU32}; use crate::loom::sync::Arc; -use crate::runtime::stats::WorkerStatsBatcher; use crate::runtime::task::{self, Inject}; +use crate::runtime::MetricsBatch; -use std::mem::MaybeUninit; +use std::mem::{self, MaybeUninit}; use std::ptr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; +// Use wider integers when possible to increase ABA resilience. +// +// See issue #5041: <https://github.com/tokio-rs/tokio/issues/5041>. +cfg_has_atomic_u64! { + type UnsignedShort = u32; + type UnsignedLong = u64; + type AtomicUnsignedShort = crate::loom::sync::atomic::AtomicU32; + type AtomicUnsignedLong = crate::loom::sync::atomic::AtomicU64; +} +cfg_not_has_atomic_u64! { + type UnsignedShort = u16; + type UnsignedLong = u32; + type AtomicUnsignedShort = crate::loom::sync::atomic::AtomicU16; + type AtomicUnsignedLong = crate::loom::sync::atomic::AtomicU32; +} + /// Producer handle. May only be used from a single thread. -pub(super) struct Local<T: 'static> { +pub(crate) struct Local<T: 'static> { inner: Arc<Inner<T>>, } /// Consumer handle. May be used from many threads. -pub(super) struct Steal<T: 'static>(Arc<Inner<T>>); +pub(crate) struct Steal<T: 'static>(Arc<Inner<T>>); -pub(super) struct Inner<T: 'static> { +pub(crate) struct Inner<T: 'static> { /// Concurrently updated by many threads. /// - /// Contains two `u16` values. The LSB byte is the "real" head of the queue. - /// The `u16` in the MSB is set by a stealer in process of stealing values. - /// It represents the first value being stolen in the batch. `u16` is used - /// in order to distinguish between `head == tail` and `head == tail - - /// capacity`. + /// Contains two `UnsignedShort` values. The LSB byte is the "real" head of + /// the queue. The `UnsignedShort` in the MSB is set by a stealer in process + /// of stealing values. It represents the first value being stolen in the + /// batch. The `UnsignedShort` indices are intentionally wider than strictly + /// required for buffer indexing in order to provide ABA mitigation and make + /// it possible to distinguish between full and empty buffers. /// - /// When both `u16` values are the same, there is no active stealer. + /// When both `UnsignedShort` values are the same, there is no active + /// stealer. /// /// Tracking an in-progress stealer prevents a wrapping scenario. - head: AtomicU32, + head: AtomicUnsignedLong, /// Only updated by producer thread but read by many threads. - tail: AtomicU16, + tail: AtomicUnsignedShort, /// Elements buffer: Box<[UnsafeCell<MaybeUninit<task::Notified<T>>>; LOCAL_QUEUE_CAPACITY]>, @@ -65,7 +82,7 @@ fn make_fixed_size<T>(buffer: Box<[T]>) -> Box<[T; LOCAL_QUEUE_CAPACITY]> { } /// Create a new local run-queue -pub(super) fn local<T: 'static>() -> (Steal<T>, Local<T>) { +pub(crate) fn local<T: 'static>() -> (Steal<T>, Local<T>) { let mut buffer = Vec::with_capacity(LOCAL_QUEUE_CAPACITY); for _ in 0..LOCAL_QUEUE_CAPACITY { @@ -73,8 +90,8 @@ pub(super) fn local<T: 'static>() -> (Steal<T>, Local<T>) { } let inner = Arc::new(Inner { - head: AtomicU32::new(0), - tail: AtomicU16::new(0), + head: AtomicUnsignedLong::new(0), + tail: AtomicUnsignedShort::new(0), buffer: make_fixed_size(buffer.into_boxed_slice()), }); @@ -88,8 +105,8 @@ pub(super) fn local<T: 'static>() -> (Steal<T>, Local<T>) { } impl<T> Local<T> { - /// Returns true if the queue has entries that can be stealed. - pub(super) fn is_stealable(&self) -> bool { + /// Returns true if the queue has entries that can be stolen. + pub(crate) fn is_stealable(&self) -> bool { !self.inner.is_empty() } @@ -97,12 +114,17 @@ impl<T> Local<T> { /// /// Separate to is_stealable so that refactors of is_stealable to "protect" /// some tasks from stealing won't affect this - pub(super) fn has_tasks(&self) -> bool { + pub(crate) fn has_tasks(&self) -> bool { !self.inner.is_empty() } /// Pushes a task to the back of the local queue, skipping the LIFO slot. - pub(super) fn push_back(&mut self, mut task: task::Notified<T>, inject: &Inject<T>) { + pub(crate) fn push_back( + &mut self, + mut task: task::Notified<T>, + inject: &Inject<T>, + metrics: &mut MetricsBatch, + ) { let tail = loop { let head = self.inner.head.load(Acquire); let (steal, real) = unpack(head); @@ -110,7 +132,7 @@ impl<T> Local<T> { // safety: this is the **only** thread that updates this cell. let tail = unsafe { self.inner.tail.unsync_load() }; - if tail.wrapping_sub(steal) < LOCAL_QUEUE_CAPACITY as u16 { + if tail.wrapping_sub(steal) < LOCAL_QUEUE_CAPACITY as UnsignedShort { // There is capacity for the task break tail; } else if steal != real { @@ -121,7 +143,7 @@ impl<T> Local<T> { } else { // Push the current task and half of the queue into the // inject queue. - match self.push_overflow(task, real, tail, inject) { + match self.push_overflow(task, real, tail, inject, metrics) { Ok(_) => return, // Lost the race, try again Err(v) => { @@ -160,15 +182,16 @@ impl<T> Local<T> { fn push_overflow( &mut self, task: task::Notified<T>, - head: u16, - tail: u16, + head: UnsignedShort, + tail: UnsignedShort, inject: &Inject<T>, + metrics: &mut MetricsBatch, ) -> Result<(), task::Notified<T>> { /// How many elements are we taking from the local queue. /// /// This is one less than the number of tasks pushed to the inject /// queue as we are also inserting the `task` argument. - const NUM_TASKS_TAKEN: u16 = (LOCAL_QUEUE_CAPACITY / 2) as u16; + const NUM_TASKS_TAKEN: UnsignedShort = (LOCAL_QUEUE_CAPACITY / 2) as UnsignedShort; assert_eq!( tail.wrapping_sub(head) as usize, @@ -213,15 +236,15 @@ impl<T> Local<T> { /// An iterator that takes elements out of the run queue. struct BatchTaskIter<'a, T: 'static> { buffer: &'a [UnsafeCell<MaybeUninit<task::Notified<T>>>; LOCAL_QUEUE_CAPACITY], - head: u32, - i: u32, + head: UnsignedLong, + i: UnsignedLong, } impl<'a, T: 'static> Iterator for BatchTaskIter<'a, T> { type Item = task::Notified<T>; #[inline] fn next(&mut self) -> Option<task::Notified<T>> { - if self.i == u32::from(NUM_TASKS_TAKEN) { + if self.i == UnsignedLong::from(NUM_TASKS_TAKEN) { None } else { let i_idx = self.i.wrapping_add(self.head) as usize & MASK; @@ -240,17 +263,20 @@ impl<T> Local<T> { // safety: The CAS above ensures that no consumer will look at these // values again, and we are the only producer. let batch_iter = BatchTaskIter { - buffer: &*self.inner.buffer, - head: head as u32, + buffer: &self.inner.buffer, + head: head as UnsignedLong, i: 0, }; inject.push_batch(batch_iter.chain(std::iter::once(task))); + // Add 1 to factor in the task currently being scheduled. + metrics.incr_overflow_count(); + Ok(()) } /// Pops a task from the local queue. - pub(super) fn pop(&mut self) -> Option<task::Notified<T>> { + pub(crate) fn pop(&mut self) -> Option<task::Notified<T>> { let mut head = self.inner.head.load(Acquire); let idx = loop { @@ -292,15 +318,15 @@ impl<T> Local<T> { } impl<T> Steal<T> { - pub(super) fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.0.is_empty() } /// Steals half the tasks from self and place them into `dst`. - pub(super) fn steal_into( + pub(crate) fn steal_into( &self, dst: &mut Local<T>, - stats: &mut WorkerStatsBatcher, + dst_metrics: &mut MetricsBatch, ) -> Option<task::Notified<T>> { // Safety: the caller is the only thread that mutates `dst.tail` and // holds a mutable reference. @@ -311,7 +337,7 @@ impl<T> Steal<T> { // from `dst` there may not be enough capacity to steal. let (steal, _) = unpack(dst.inner.head.load(Acquire)); - if dst_tail.wrapping_sub(steal) > LOCAL_QUEUE_CAPACITY as u16 / 2 { + if dst_tail.wrapping_sub(steal) > LOCAL_QUEUE_CAPACITY as UnsignedShort / 2 { // we *could* try to steal less here, but for simplicity, we're just // going to abort. return None; @@ -320,13 +346,15 @@ impl<T> Steal<T> { // Steal the tasks into `dst`'s buffer. This does not yet expose the // tasks in `dst`. let mut n = self.steal_into2(dst, dst_tail); - stats.incr_steal_count(n); if n == 0 { // No tasks were stolen return None; } + dst_metrics.incr_steal_count(n as u16); + dst_metrics.incr_steal_operations(); + // We are returning a task here n -= 1; @@ -350,7 +378,7 @@ impl<T> Steal<T> { // Steal tasks from `self`, placing them into `dst`. Returns the number of // tasks that were stolen. - fn steal_into2(&self, dst: &mut Local<T>, dst_tail: u16) -> u16 { + fn steal_into2(&self, dst: &mut Local<T>, dst_tail: UnsignedShort) -> UnsignedShort { let mut prev_packed = self.0.head.load(Acquire); let mut next_packed; @@ -392,7 +420,11 @@ impl<T> Steal<T> { } }; - assert!(n <= LOCAL_QUEUE_CAPACITY as u16 / 2, "actual = {}", n); + assert!( + n <= LOCAL_QUEUE_CAPACITY as UnsignedShort / 2, + "actual = {}", + n + ); let (first, _) = unpack(next_packed); @@ -446,6 +478,14 @@ impl<T> Steal<T> { } } +cfg_metrics! { + impl<T> Steal<T> { + pub(crate) fn len(&self) -> usize { + self.0.len() as _ + } + } +} + impl<T> Clone for Steal<T> { fn clone(&self) -> Steal<T> { Steal(self.0.clone()) @@ -461,26 +501,30 @@ impl<T> Drop for Local<T> { } impl<T> Inner<T> { - fn is_empty(&self) -> bool { + fn len(&self) -> UnsignedShort { let (_, head) = unpack(self.head.load(Acquire)); let tail = self.tail.load(Acquire); - head == tail + tail.wrapping_sub(head) + } + + fn is_empty(&self) -> bool { + self.len() == 0 } } /// Split the head value into the real head and the index a stealer is working /// on. -fn unpack(n: u32) -> (u16, u16) { - let real = n & u16::MAX as u32; - let steal = n >> 16; +fn unpack(n: UnsignedLong) -> (UnsignedShort, UnsignedShort) { + let real = n & UnsignedShort::MAX as UnsignedLong; + let steal = n >> (mem::size_of::<UnsignedShort>() * 8); - (steal as u16, real as u16) + (steal as UnsignedShort, real as UnsignedShort) } /// Join the two head values -fn pack(steal: u16, real: u16) -> u32 { - (real as u32) | ((steal as u32) << 16) +fn pack(steal: UnsignedShort, real: UnsignedShort) -> UnsignedLong { + (real as UnsignedLong) | ((steal as UnsignedLong) << (mem::size_of::<UnsignedShort>() * 8)) } #[test] diff --git a/src/runtime/thread_pool/worker.rs b/src/runtime/scheduler/multi_thread/worker.rs index ae8efe6..148255a 100644 --- a/src/runtime/thread_pool/worker.rs +++ b/src/runtime/scheduler/multi_thread/worker.rs @@ -56,27 +56,24 @@ //! the inject queue indefinitely. This would be a ref-count cycle and a memory //! leak. -use crate::coop; -use crate::future::Future; -use crate::loom::rand::seed; use crate::loom::sync::{Arc, Mutex}; -use crate::park::{Park, Unpark}; use crate::runtime; -use crate::runtime::enter::EnterContext; -use crate::runtime::park::{Parker, Unparker}; -use crate::runtime::stats::{RuntimeStats, WorkerStatsBatcher}; -use crate::runtime::task::{Inject, JoinHandle, OwnedTasks}; -use crate::runtime::thread_pool::{AtomicCell, Idle}; -use crate::runtime::{queue, task, Callback}; -use crate::util::FastRand; +use crate::runtime::context; +use crate::runtime::scheduler::multi_thread::{queue, Handle, Idle, Parker, Unparker}; +use crate::runtime::task::{Inject, OwnedTasks}; +use crate::runtime::{ + blocking, coop, driver, scheduler, task, Config, MetricsBatch, SchedulerMetrics, WorkerMetrics, +}; +use crate::util::atomic_cell::AtomicCell; +use crate::util::rand::{FastRand, RngSeedGenerator}; use std::cell::RefCell; use std::time::Duration; /// A scheduler worker pub(super) struct Worker { - /// Reference to shared state - shared: Arc<Shared>, + /// Reference to scheduler's handle + handle: Arc<Handle>, /// Index holding this worker's remote state index: usize, @@ -88,7 +85,7 @@ pub(super) struct Worker { /// Core data struct Core { /// Used to schedule bookkeeping tasks every so often. - tick: u8, + tick: u32, /// When a task is scheduled from a worker, it is stored in this slot. The /// worker will check this slot for a task **before** checking the run @@ -98,7 +95,7 @@ struct Core { lifo_slot: Option<Notified>, /// The worker-local run queue. - run_queue: queue::Local<Arc<Shared>>, + run_queue: queue::Local<Arc<Handle>>, /// True if the worker is currently searching for more work. Searching /// involves attempting to steal from other workers. @@ -113,8 +110,8 @@ struct Core { /// borrow checker happy. park: Option<Parker>, - /// Batching stats so they can be submitted to RuntimeStats. - stats: WorkerStatsBatcher, + /// Batching metrics so they can be submitted to RuntimeMetrics. + metrics: MetricsBatch, /// Fast random number generator. rand: FastRand, @@ -126,14 +123,16 @@ pub(super) struct Shared { /// how they communicate between each other. remotes: Box<[Remote]>, - /// Submits work to the scheduler while **not** currently on a worker thread. - inject: Inject<Arc<Shared>>, + /// Global task queue used for: + /// 1. Submit work to the scheduler while **not** currently on a worker thread. + /// 2. Submit work to the scheduler when a worker run queue is saturated + inject: Inject<Arc<Handle>>, /// Coordinates idle workers idle: Idle, /// Collection of all active tasks spawned onto this executor. - owned: OwnedTasks<Arc<Shared>>, + pub(super) owned: OwnedTasks<Arc<Handle>>, /// Cores that have observed the shutdown signal /// @@ -142,19 +141,19 @@ pub(super) struct Shared { #[allow(clippy::vec_box)] // we're moving an already-boxed value shutdown_cores: Mutex<Vec<Box<Core>>>, - /// Callback for a worker parking itself - before_park: Option<Callback>, - /// Callback for a worker unparking itself - after_unpark: Option<Callback>, + /// Scheduler configuration options + config: Config, - /// Collects stats from the runtime. - stats: RuntimeStats, + /// Collects metrics from the runtime. + pub(super) scheduler_metrics: SchedulerMetrics, + + pub(super) worker_metrics: Box<[WorkerMetrics]>, } /// Used to communicate with a worker from other threads. struct Remote { /// Steals tasks from this worker. - steal: queue::Steal<Arc<Shared>>, + steal: queue::Steal<Arc<Handle>>, /// Unparks the associated worker thread unpark: Unparker, @@ -178,10 +177,10 @@ pub(crate) struct Launch(Vec<Arc<Worker>>); type RunResult = Result<Box<Core>, ()>; /// A task handle -type Task = task::Task<Arc<Shared>>; +type Task = task::Task<Arc<Handle>>; /// A notified task handle -type Notified = task::Notified<Arc<Shared>>; +type Notified = task::Notified<Arc<Handle>>; // Tracks thread-local state scoped_thread_local!(static CURRENT: Context); @@ -189,14 +188,17 @@ scoped_thread_local!(static CURRENT: Context); pub(super) fn create( size: usize, park: Parker, - before_park: Option<Callback>, - after_unpark: Option<Callback>, -) -> (Arc<Shared>, Launch) { - let mut cores = vec![]; - let mut remotes = vec![]; + driver_handle: driver::Handle, + blocking_spawner: blocking::Spawner, + seed_generator: RngSeedGenerator, + config: Config, +) -> (Arc<Handle>, Launch) { + let mut cores = Vec::with_capacity(size); + let mut remotes = Vec::with_capacity(size); + let mut worker_metrics = Vec::with_capacity(size); // Create the local queues - for i in 0..size { + for _ in 0..size { let (steal, run_queue) = queue::local(); let park = park.clone(); @@ -209,37 +211,44 @@ pub(super) fn create( is_searching: false, is_shutdown: false, park: Some(park), - stats: WorkerStatsBatcher::new(i), - rand: FastRand::new(seed()), + metrics: MetricsBatch::new(), + rand: FastRand::new(config.seed_generator.next_seed()), })); remotes.push(Remote { steal, unpark }); - } - - let shared = Arc::new(Shared { - remotes: remotes.into_boxed_slice(), - inject: Inject::new(), - idle: Idle::new(size), - owned: OwnedTasks::new(), - shutdown_cores: Mutex::new(vec![]), - before_park, - after_unpark, - stats: RuntimeStats::new(size), + worker_metrics.push(WorkerMetrics::new()); + } + + let handle = Arc::new(Handle { + shared: Shared { + remotes: remotes.into_boxed_slice(), + inject: Inject::new(), + idle: Idle::new(size), + owned: OwnedTasks::new(), + shutdown_cores: Mutex::new(vec![]), + config, + scheduler_metrics: SchedulerMetrics::new(), + worker_metrics: worker_metrics.into_boxed_slice(), + }, + driver: driver_handle, + blocking_spawner, + seed_generator, }); let mut launch = Launch(vec![]); for (index, core) in cores.drain(..).enumerate() { launch.0.push(Arc::new(Worker { - shared: shared.clone(), + handle: handle.clone(), index, core: AtomicCell::new(Some(core)), })); } - (shared, launch) + (handle, launch) } +#[track_caller] pub(crate) fn block_in_place<F, R>(f: F) -> R where F: FnOnce() -> R, @@ -266,35 +275,45 @@ where let mut had_entered = false; - CURRENT.with(|maybe_cx| { - match (crate::runtime::enter::context(), maybe_cx.is_some()) { - (EnterContext::Entered { .. }, true) => { + let setup_result = CURRENT.with(|maybe_cx| { + match ( + crate::runtime::context::current_enter_context(), + maybe_cx.is_some(), + ) { + (context::EnterRuntime::Entered { .. }, true) => { // We are on a thread pool runtime thread, so we just need to // set up blocking. had_entered = true; } - (EnterContext::Entered { allow_blocking }, false) => { + ( + context::EnterRuntime::Entered { + allow_block_in_place, + }, + false, + ) => { // We are on an executor, but _not_ on the thread pool. That is // _only_ okay if we are in a thread pool runtime's block_on // method: - if allow_blocking { + if allow_block_in_place { had_entered = true; - return; + return Ok(()); } else { - // This probably means we are on the basic_scheduler or in a + // This probably means we are on the current_thread runtime or in a // LocalSet, where it is _not_ okay to block. - panic!("can call blocking only when running on the multi-threaded runtime"); + return Err( + "can call blocking only when running on the multi-threaded runtime", + ); } } - (EnterContext::NotEntered, true) => { + (context::EnterRuntime::NotEntered, true) => { // This is a nested call to block_in_place (we already exited). // All the necessary setup has already been done. - return; + return Ok(()); } - (EnterContext::NotEntered, false) => { + (context::EnterRuntime::NotEntered, false) => { // We are outside of the tokio runtime, so blocking is fine. // We can also skip all of the thread pool blocking setup steps. - return; + return Ok(()); } } @@ -303,7 +322,7 @@ where // Get the worker core. If none is set, then blocking is fine! let core = match cx.core.borrow_mut().take() { Some(core) => core, - None => return, + None => return Ok(()), }; // The parker should be set here @@ -322,25 +341,24 @@ where // steal the core back. let worker = cx.worker.clone(); runtime::spawn_blocking(move || run(worker)); + Ok(()) }); + if let Err(panic_message) = setup_result { + panic!("{}", panic_message); + } + if had_entered { // Unset the current task's budget. Blocking sections are not // constrained by task budgets. let _reset = Reset(coop::stop()); - crate::runtime::enter::exit(f) + crate::runtime::context::exit_runtime(f) } else { f() } } -/// After how many ticks is the global queue polled. This helps to ensure -/// fairness. -/// -/// The number is fairly arbitrary. I believe this value was copied from golang. -const GLOBAL_POLL_INTERVAL: u8 = 61; - impl Launch { pub(crate) fn launch(mut self) { for worker in self.0.drain(..) { @@ -350,6 +368,22 @@ impl Launch { } fn run(worker: Arc<Worker>) { + struct AbortOnPanic; + + impl Drop for AbortOnPanic { + fn drop(&mut self) { + if std::thread::panicking() { + eprintln!("worker thread panicking; aborting process"); + std::process::abort(); + } + } + } + + // Catching panics on worker threads in tests is quite tricky. Instead, when + // debug assertions are enabled, we just abort the process. + #[cfg(debug_assertions)] + let _abort_on_panic = AbortOnPanic; + // Acquire a core. If this fails, then another thread is running this // worker and there is nothing further to do. let core = match worker.core.take() { @@ -357,18 +391,24 @@ fn run(worker: Arc<Worker>) { None => return, }; + let handle = scheduler::Handle::MultiThread(worker.handle.clone()); + let _enter = crate::runtime::context::enter_runtime(&handle, true); + // Set the worker context. let cx = Context { worker, core: RefCell::new(None), }; - let _enter = crate::runtime::enter(true); - CURRENT.set(&cx, || { // This should always be an error. It only returns a `Result` to support // using `?` to short circuit. assert!(cx.run(core).is_err()); + + // Check if there are any deferred tasks to notify. This can happen when + // the worker core is lost due to `block_in_place()` being called from + // within the task. + wake_deferred_tasks(); }); } @@ -393,26 +433,30 @@ impl Context { core = self.run_task(task, core)?; } else { // Wait for work - core = self.park(core); + core = if did_defer_tasks() { + self.park_timeout(core, Some(Duration::from_millis(0))) + } else { + self.park(core) + }; } } core.pre_shutdown(&self.worker); // Signal shutdown - self.worker.shared.shutdown(core); + self.worker.handle.shutdown_core(core); Err(()) } fn run_task(&self, task: Notified, mut core: Box<Core>) -> RunResult { - let task = self.worker.shared.owned.assert_owner(task); + let task = self.worker.handle.shared.owned.assert_owner(task); // Make sure the worker is not in the **searching** state. This enables // another idle worker to try to steal work. core.transition_from_searching(&self.worker); // Make the core available to the runtime context - core.stats.incr_poll_count(); + core.metrics.incr_poll_count(); *self.core.borrow_mut() = Some(core); // Run the task @@ -437,14 +481,15 @@ impl Context { if coop::has_budget_remaining() { // Run the LIFO task, then loop - core.stats.incr_poll_count(); + core.metrics.incr_poll_count(); *self.core.borrow_mut() = Some(core); - let task = self.worker.shared.owned.assert_owner(task); + let task = self.worker.handle.shared.owned.assert_owner(task); task.run(); } else { // Not enough budget left to run the LIFO task, push it to // the back of the queue and return. - core.run_queue.push_back(task, self.worker.inject()); + core.run_queue + .push_back(task, self.worker.inject(), &mut core.metrics); return Ok(core); } } @@ -452,7 +497,7 @@ impl Context { } fn maintenance(&self, mut core: Box<Core>) -> Box<Core> { - if core.tick % GLOBAL_POLL_INTERVAL == 0 { + if core.tick % self.worker.handle.shared.config.event_interval == 0 { // Call `park` with a 0 timeout. This enables the I/O driver, timer, ... // to run without actually putting the thread to sleep. core = self.park_timeout(core, Some(Duration::from_millis(0))); @@ -464,14 +509,27 @@ impl Context { core } + /// Parks the worker thread while waiting for tasks to execute. + /// + /// This function checks if indeed there's no more work left to be done before parking. + /// Also important to notice that, before parking, the worker thread will try to take + /// ownership of the Driver (IO/Time) and dispatch any events that might have fired. + /// Whenever a worker thread executes the Driver loop, all waken tasks are scheduled + /// in its own local queue until the queue saturates (ntasks > LOCAL_QUEUE_CAPACITY). + /// When the local queue is saturated, the overflow tasks are added to the injection queue + /// from where other workers can pick them up. + /// Also, we rely on the workstealing algorithm to spread the tasks amongst workers + /// after all the IOs get dispatched fn park(&self, mut core: Box<Core>) -> Box<Core> { - if let Some(f) = &self.worker.shared.before_park { + if let Some(f) = &self.worker.handle.shared.config.before_park { f(); } if core.transition_to_parked(&self.worker) { while !core.is_shutdown { + core.metrics.about_to_park(); core = self.park_timeout(core, None); + core.metrics.returned_from_park(); // Run regularly scheduled maintenance core.maintenance(&self.worker); @@ -482,7 +540,7 @@ impl Context { } } - if let Some(f) = &self.worker.shared.after_unpark { + if let Some(f) = &self.worker.handle.shared.config.after_unpark { f(); } core @@ -492,31 +550,30 @@ impl Context { // Take the parker out of core let mut park = core.park.take().expect("park missing"); - core.stats.about_to_park(); - // Store `core` in context *self.core.borrow_mut() = Some(core); // Park thread if let Some(timeout) = duration { - park.park_timeout(timeout).expect("park failed"); + park.park_timeout(&self.worker.handle.driver, timeout); } else { - park.park().expect("park failed"); + park.park(&self.worker.handle.driver); } + wake_deferred_tasks(); + // Remove `core` from context core = self.core.borrow_mut().take().expect("core missing"); // Place `park` back in `core` core.park = Some(park); - // If there are tasks available to steal, notify a worker - if core.run_queue.is_stealable() { - self.worker.shared.notify_parked(); + // If there are tasks available to steal, but this worker is not + // looking for tasks to steal, notify another worker. + if !core.is_searching && core.run_queue.is_stealable() { + self.worker.handle.notify_parked(); } - core.stats.returned_from_park(); - core } } @@ -529,7 +586,7 @@ impl Core { /// Return the next notified task available to this worker. fn next_task(&mut self, worker: &Worker) -> Option<Notified> { - if self.tick % GLOBAL_POLL_INTERVAL == 0 { + if self.tick % worker.handle.shared.config.global_queue_interval == 0 { worker.inject().pop().or_else(|| self.next_local_task()) } else { self.next_local_task().or_else(|| worker.inject().pop()) @@ -540,12 +597,17 @@ impl Core { self.lifo_slot.take().or_else(|| self.run_queue.pop()) } + /// Function responsible for stealing tasks from another worker + /// + /// Note: Only if less than half the workers are searching for tasks to steal + /// a new worker will actually try to steal. The idea is to make sure not all + /// workers will be trying to steal at the same time. fn steal_work(&mut self, worker: &Worker) -> Option<Notified> { if !self.transition_to_searching(worker) { return None; } - let num = worker.shared.remotes.len(); + let num = worker.handle.shared.remotes.len(); // Start from a random worker let start = self.rand.fastrand_n(num as u32) as usize; @@ -557,22 +619,22 @@ impl Core { continue; } - let target = &worker.shared.remotes[i]; + let target = &worker.handle.shared.remotes[i]; if let Some(task) = target .steal - .steal_into(&mut self.run_queue, &mut self.stats) + .steal_into(&mut self.run_queue, &mut self.metrics) { return Some(task); } } // Fallback on checking the global queue - worker.shared.inject.pop() + worker.handle.shared.inject.pop() } fn transition_to_searching(&mut self, worker: &Worker) -> bool { if !self.is_searching { - self.is_searching = worker.shared.idle.transition_worker_to_searching(); + self.is_searching = worker.handle.shared.idle.transition_worker_to_searching(); } self.is_searching @@ -584,12 +646,12 @@ impl Core { } self.is_searching = false; - worker.shared.transition_worker_from_searching(); + worker.handle.transition_worker_from_searching(); } /// Prepares the worker state for parking. /// - /// Returns true if the transition happend, false if there is work to do first. + /// Returns true if the transition happened, false if there is work to do first. fn transition_to_parked(&mut self, worker: &Worker) -> bool { // Workers should not park if they have work to do if self.lifo_slot.is_some() || self.run_queue.has_tasks() { @@ -600,6 +662,7 @@ impl Core { // must check all the queues one last time in case work materialized // between the last work scan and transitioning out of searching. let is_last_searcher = worker + .handle .shared .idle .transition_worker_to_parked(worker.index, self.is_searching); @@ -609,7 +672,7 @@ impl Core { self.is_searching = false; if is_last_searcher { - worker.shared.notify_if_work_pending(); + worker.handle.notify_if_work_pending(); } true @@ -620,12 +683,15 @@ impl Core { // If a task is in the lifo slot, then we must unpark regardless of // being notified if self.lifo_slot.is_some() { - worker.shared.idle.unpark_worker_by_id(worker.index); - self.is_searching = true; + // When a worker wakes, it should only transition to the "searching" + // state when the wake originates from another worker *or* a new task + // is pushed. We do *not* want the worker to transition to "searching" + // when it wakes when the I/O driver receives new events. + self.is_searching = !worker.handle.shared.idle.unpark_worker_by_id(worker.index); return true; } - if worker.shared.idle.is_parked(worker.index) { + if worker.handle.shared.idle.is_parked(worker.index) { return false; } @@ -636,7 +702,8 @@ impl Core { /// Runs maintenance work such as checking the pool's state. fn maintenance(&mut self, worker: &Worker) { - self.stats.submit(&worker.shared.stats); + self.metrics + .submit(&worker.handle.shared.worker_metrics[worker.index]); if !self.is_shutdown { // Check if the scheduler has been shutdown @@ -648,68 +715,52 @@ impl Core { /// before we enter the single-threaded phase of shutdown processing. fn pre_shutdown(&mut self, worker: &Worker) { // Signal to all tasks to shut down. - worker.shared.owned.close_and_shutdown_all(); + worker.handle.shared.owned.close_and_shutdown_all(); - self.stats.submit(&worker.shared.stats); + self.metrics + .submit(&worker.handle.shared.worker_metrics[worker.index]); } /// Shuts down the core. - fn shutdown(&mut self) { + fn shutdown(&mut self, handle: &Handle) { // Take the core let mut park = self.park.take().expect("park missing"); // Drain the queue while self.next_local_task().is_some() {} - park.shutdown(); + park.shutdown(&handle.driver); } } impl Worker { /// Returns a reference to the scheduler's injection queue. - fn inject(&self) -> &Inject<Arc<Shared>> { - &self.shared.inject + fn inject(&self) -> &Inject<Arc<Handle>> { + &self.handle.shared.inject } } -impl task::Schedule for Arc<Shared> { +// TODO: Move `Handle` impls into handle.rs +impl task::Schedule for Arc<Handle> { fn release(&self, task: &Task) -> Option<Task> { - self.owned.remove(task) + self.shared.owned.remove(task) } fn schedule(&self, task: Notified) { - (**self).schedule(task, false); + self.schedule_task(task, false); } fn yield_now(&self, task: Notified) { - (**self).schedule(task, true); + self.schedule_task(task, true); } } -impl Shared { - pub(super) fn bind_new_task<T>(me: &Arc<Self>, future: T) -> JoinHandle<T::Output> - where - T: Future + Send + 'static, - T::Output: Send + 'static, - { - let (handle, notified) = me.owned.bind(future, me.clone()); - - if let Some(notified) = notified { - me.schedule(notified, false); - } - - handle - } - - pub(crate) fn stats(&self) -> &RuntimeStats { - &self.stats - } - - pub(super) fn schedule(&self, task: Notified, is_yield: bool) { +impl Handle { + pub(super) fn schedule_task(&self, task: Notified, is_yield: bool) { CURRENT.with(|maybe_cx| { if let Some(cx) = maybe_cx { // Make sure the task is part of the **current** scheduler. - if self.ptr_eq(&cx.worker.shared) { + if self.ptr_eq(&cx.worker.handle) { // And the current thread still holds a core if let Some(core) = cx.core.borrow_mut().as_mut() { self.schedule_local(core, task, is_yield); @@ -719,18 +770,22 @@ impl Shared { } // Otherwise, use the inject queue. - self.inject.push(task); + self.shared.inject.push(task); + self.shared.scheduler_metrics.inc_remote_schedule_count(); self.notify_parked(); }) } fn schedule_local(&self, core: &mut Core, task: Notified, is_yield: bool) { + core.metrics.inc_local_schedule_count(); + // Spawning from the worker thread. If scheduling a "yield" then the // task must always be pushed to the back of the queue, enabling other // tasks to be executed. If **not** a yield, then there is more // flexibility and the task may go to the front of the queue. - let should_notify = if is_yield { - core.run_queue.push_back(task, &self.inject); + let should_notify = if is_yield || self.shared.config.disable_lifo_slot { + core.run_queue + .push_back(task, &self.shared.inject, &mut core.metrics); true } else { // Push to the LIFO slot @@ -738,7 +793,8 @@ impl Shared { let ret = prev.is_some(); if let Some(prev) = prev { - core.run_queue.push_back(prev, &self.inject); + core.run_queue + .push_back(prev, &self.shared.inject, &mut core.metrics); } core.lifo_slot = Some(task); @@ -755,38 +811,38 @@ impl Shared { } pub(super) fn close(&self) { - if self.inject.close() { + if self.shared.inject.close() { self.notify_all(); } } fn notify_parked(&self) { - if let Some(index) = self.idle.worker_to_notify() { - self.remotes[index].unpark.unpark(); + if let Some(index) = self.shared.idle.worker_to_notify() { + self.shared.remotes[index].unpark.unpark(&self.driver); } } fn notify_all(&self) { - for remote in &self.remotes[..] { - remote.unpark.unpark(); + for remote in &self.shared.remotes[..] { + remote.unpark.unpark(&self.driver); } } fn notify_if_work_pending(&self) { - for remote in &self.remotes[..] { + for remote in &self.shared.remotes[..] { if !remote.steal.is_empty() { self.notify_parked(); return; } } - if !self.inject.is_empty() { + if !self.shared.inject.is_empty() { self.notify_parked(); } } fn transition_worker_from_searching(&self) { - if self.idle.transition_worker_from_searching() { + if self.shared.idle.transition_worker_from_searching() { // We are the final searching worker. Because work was found, we // need to notify another worker. self.notify_parked(); @@ -797,29 +853,49 @@ impl Shared { /// its core back into its handle. /// /// If all workers have reached this point, the final cleanup is performed. - fn shutdown(&self, core: Box<Core>) { - let mut cores = self.shutdown_cores.lock(); + fn shutdown_core(&self, core: Box<Core>) { + let mut cores = self.shared.shutdown_cores.lock(); cores.push(core); - if cores.len() != self.remotes.len() { + if cores.len() != self.shared.remotes.len() { return; } - debug_assert!(self.owned.is_empty()); + debug_assert!(self.shared.owned.is_empty()); for mut core in cores.drain(..) { - core.shutdown(); + core.shutdown(self); } // Drain the injection queue // // We already shut down every task, so we can simply drop the tasks. - while let Some(task) = self.inject.pop() { + while let Some(task) = self.shared.inject.pop() { drop(task); } } - fn ptr_eq(&self, other: &Shared) -> bool { + fn ptr_eq(&self, other: &Handle) -> bool { std::ptr::eq(self, other) } } + +fn did_defer_tasks() -> bool { + context::with_defer(|deferred| !deferred.is_empty()).unwrap() +} + +fn wake_deferred_tasks() { + context::with_defer(|deferred| deferred.wake()); +} + +cfg_metrics! { + impl Shared { + pub(super) fn injection_queue_depth(&self) -> usize { + self.inject.len() + } + + pub(super) fn worker_local_queue_depth(&self, worker: usize) -> usize { + self.remotes[worker].steal.len() + } + } +} diff --git a/src/runtime/signal/mod.rs b/src/runtime/signal/mod.rs new file mode 100644 index 0000000..24f2f4c --- /dev/null +++ b/src/runtime/signal/mod.rs @@ -0,0 +1,142 @@ +#![cfg_attr(not(feature = "rt"), allow(dead_code))] + +//! Signal driver + +use crate::runtime::{driver, io}; +use crate::signal::registry::globals; + +use mio::net::UnixStream; +use std::io::{self as std_io, Read}; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +/// Responsible for registering wakeups when an OS signal is received, and +/// subsequently dispatching notifications to any signal listeners as appropriate. +/// +/// Note: this driver relies on having an enabled IO driver in order to listen to +/// pipe write wakeups. +#[derive(Debug)] +pub(crate) struct Driver { + /// Thread parker. The `Driver` park implementation delegates to this. + io: io::Driver, + + /// A pipe for receiving wake events from the signal handler + receiver: UnixStream, + + /// Shared state. The driver keeps a strong ref and the handle keeps a weak + /// ref. The weak ref is used to check if the driver is still active before + /// trying to register a signal handler. + inner: Arc<()>, +} + +#[derive(Debug, Default)] +pub(crate) struct Handle { + /// Paired w/ the `Arc` above and is used to check if the driver is still + /// around before attempting to register a signal handler. + inner: Weak<()>, +} + +// ===== impl Driver ===== + +impl Driver { + /// Creates a new signal `Driver` instance that delegates wakeups to `park`. + pub(crate) fn new(io: io::Driver, io_handle: &io::Handle) -> std_io::Result<Self> { + use std::mem::ManuallyDrop; + use std::os::unix::io::{AsRawFd, FromRawFd}; + + // NB: We give each driver a "fresh" receiver file descriptor to avoid + // the issues described in alexcrichton/tokio-process#42. + // + // In the past we would reuse the actual receiver file descriptor and + // swallow any errors around double registration of the same descriptor. + // I'm not sure if the second (failed) registration simply doesn't end + // up receiving wake up notifications, or there could be some race + // condition when consuming readiness events, but having distinct + // descriptors appears to mitigate this. + // + // Unfortunately we cannot just use a single global UnixStream instance + // either, since we can't assume they will always be registered with the + // exact same reactor. + // + // Mio 0.7 removed `try_clone()` as an API due to unexpected behavior + // with registering dups with the same reactor. In this case, duping is + // safe as each dup is registered with separate reactors **and** we + // only expect at least one dup to receive the notification. + + // Manually drop as we don't actually own this instance of UnixStream. + let receiver_fd = globals().receiver.as_raw_fd(); + + // safety: there is nothing unsafe about this, but the `from_raw_fd` fn is marked as unsafe. + let original = + ManuallyDrop::new(unsafe { std::os::unix::net::UnixStream::from_raw_fd(receiver_fd) }); + let mut receiver = UnixStream::from_std(original.try_clone()?); + + io_handle.register_signal_receiver(&mut receiver)?; + + Ok(Self { + io, + receiver, + inner: Arc::new(()), + }) + } + + /// Returns a handle to this event loop which can be sent across threads + /// and can be used as a proxy to the event loop itself. + pub(crate) fn handle(&self) -> Handle { + Handle { + inner: Arc::downgrade(&self.inner), + } + } + + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.io.park(handle); + self.process(); + } + + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + self.io.park_timeout(handle, duration); + self.process(); + } + + pub(crate) fn shutdown(&mut self, handle: &driver::Handle) { + self.io.shutdown(handle) + } + + fn process(&mut self) { + // If the signal pipe has not received a readiness event, then there is + // nothing else to do. + if !self.io.consume_signal_ready() { + return; + } + + // Drain the pipe completely so we can receive a new readiness event + // if another signal has come in. + let mut buf = [0; 128]; + loop { + match self.receiver.read(&mut buf) { + Ok(0) => panic!("EOF on self-pipe"), + Ok(_) => continue, // Keep reading + Err(e) if e.kind() == std_io::ErrorKind::WouldBlock => break, + Err(e) => panic!("Bad read on self-pipe: {}", e), + } + } + + // Broadcast any signals which were received + globals().broadcast(); + } +} + +// ===== impl Handle ===== + +impl Handle { + pub(crate) fn check_inner(&self) -> std_io::Result<()> { + if self.inner.strong_count() > 0 { + Ok(()) + } else { + Err(std_io::Error::new( + std_io::ErrorKind::Other, + "signal driver gone", + )) + } + } +} diff --git a/src/runtime/spawner.rs b/src/runtime/spawner.rs deleted file mode 100644 index 9a3d465..0000000 --- a/src/runtime/spawner.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::future::Future; -use crate::runtime::basic_scheduler; -use crate::runtime::stats::RuntimeStats; -use crate::task::JoinHandle; - -cfg_rt_multi_thread! { - use crate::runtime::thread_pool; -} - -#[derive(Debug, Clone)] -pub(crate) enum Spawner { - Basic(basic_scheduler::Spawner), - #[cfg(feature = "rt-multi-thread")] - ThreadPool(thread_pool::Spawner), -} - -impl Spawner { - pub(crate) fn shutdown(&mut self) { - #[cfg(feature = "rt-multi-thread")] - { - if let Spawner::ThreadPool(spawner) = self { - spawner.shutdown(); - } - } - } - - pub(crate) fn spawn<F>(&self, future: F) -> JoinHandle<F::Output> - where - F: Future + Send + 'static, - F::Output: Send + 'static, - { - match self { - Spawner::Basic(spawner) => spawner.spawn(future), - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.spawn(future), - } - } - - #[cfg_attr(not(all(tokio_unstable, feature = "stats")), allow(dead_code))] - pub(crate) fn stats(&self) -> &RuntimeStats { - match self { - Spawner::Basic(spawner) => spawner.stats(), - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.stats(), - } - } -} diff --git a/src/runtime/stats/mock.rs b/src/runtime/stats/mock.rs deleted file mode 100644 index 3bda8bf..0000000 --- a/src/runtime/stats/mock.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! This file contains mocks of the types in src/runtime/stats/stats.rs - -pub(crate) struct RuntimeStats {} - -impl RuntimeStats { - pub(crate) fn new(_worker_threads: usize) -> Self { - Self {} - } -} - -pub(crate) struct WorkerStatsBatcher {} - -impl WorkerStatsBatcher { - pub(crate) fn new(_my_index: usize) -> Self { - Self {} - } - - pub(crate) fn submit(&mut self, _to: &RuntimeStats) {} - - pub(crate) fn about_to_park(&mut self) {} - pub(crate) fn returned_from_park(&mut self) {} - - #[cfg(feature = "rt-multi-thread")] - pub(crate) fn incr_steal_count(&mut self, _by: u16) {} - - pub(crate) fn incr_poll_count(&mut self) {} -} diff --git a/src/runtime/stats/mod.rs b/src/runtime/stats/mod.rs deleted file mode 100644 index 5e08e8e..0000000 --- a/src/runtime/stats/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! This module contains information need to view information about how the -//! runtime is performing. -#![allow(clippy::module_inception)] - -cfg_stats! { - mod stats; - - pub use self::stats::{RuntimeStats, WorkerStats}; - pub(crate) use self::stats::WorkerStatsBatcher; -} - -cfg_not_stats! { - #[path = "mock.rs"] - mod stats; - - pub(crate) use self::stats::{RuntimeStats, WorkerStatsBatcher}; -} diff --git a/src/runtime/stats/stats.rs b/src/runtime/stats/stats.rs deleted file mode 100644 index b2bcacc..0000000 --- a/src/runtime/stats/stats.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! This file contains the types necessary to collect various types of stats. -use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; - -use std::convert::TryFrom; -use std::time::{Duration, Instant}; - -/// This type contains methods to retrieve stats from a Tokio runtime. -#[derive(Debug)] -pub struct RuntimeStats { - workers: Box<[WorkerStats]>, -} - -/// This type contains methods to retrieve stats from a worker thread on a Tokio runtime. -#[derive(Debug)] -#[repr(align(128))] -pub struct WorkerStats { - park_count: AtomicU64, - steal_count: AtomicU64, - poll_count: AtomicU64, - busy_duration_total: AtomicU64, -} - -impl RuntimeStats { - pub(crate) fn new(worker_threads: usize) -> Self { - let mut workers = Vec::with_capacity(worker_threads); - for _ in 0..worker_threads { - workers.push(WorkerStats { - park_count: AtomicU64::new(0), - steal_count: AtomicU64::new(0), - poll_count: AtomicU64::new(0), - busy_duration_total: AtomicU64::new(0), - }); - } - - Self { - workers: workers.into_boxed_slice(), - } - } - - /// Returns a slice containing the worker stats for each worker thread. - pub fn workers(&self) -> impl Iterator<Item = &WorkerStats> { - self.workers.iter() - } -} - -impl WorkerStats { - /// Returns the total number of times this worker thread has parked. - pub fn park_count(&self) -> u64 { - self.park_count.load(Relaxed) - } - - /// Returns the number of tasks this worker has stolen from other worker - /// threads. - pub fn steal_count(&self) -> u64 { - self.steal_count.load(Relaxed) - } - - /// Returns the number of times this worker has polled a task. - pub fn poll_count(&self) -> u64 { - self.poll_count.load(Relaxed) - } - - /// Returns the total amount of time this worker has been busy for. - pub fn total_busy_duration(&self) -> Duration { - Duration::from_nanos(self.busy_duration_total.load(Relaxed)) - } -} - -pub(crate) struct WorkerStatsBatcher { - my_index: usize, - park_count: u64, - steal_count: u64, - poll_count: u64, - /// The total busy duration in nanoseconds. - busy_duration_total: u64, - last_resume_time: Instant, -} - -impl WorkerStatsBatcher { - pub(crate) fn new(my_index: usize) -> Self { - Self { - my_index, - park_count: 0, - steal_count: 0, - poll_count: 0, - busy_duration_total: 0, - last_resume_time: Instant::now(), - } - } - pub(crate) fn submit(&mut self, to: &RuntimeStats) { - let worker = &to.workers[self.my_index]; - - worker.park_count.store(self.park_count, Relaxed); - worker.steal_count.store(self.steal_count, Relaxed); - worker.poll_count.store(self.poll_count, Relaxed); - - worker - .busy_duration_total - .store(self.busy_duration_total, Relaxed); - } - - pub(crate) fn about_to_park(&mut self) { - self.park_count += 1; - - let busy_duration = self.last_resume_time.elapsed(); - let busy_duration = u64::try_from(busy_duration.as_nanos()).unwrap_or(u64::MAX); - self.busy_duration_total += busy_duration; - } - - pub(crate) fn returned_from_park(&mut self) { - self.last_resume_time = Instant::now(); - } - - #[cfg(feature = "rt-multi-thread")] - pub(crate) fn incr_steal_count(&mut self, by: u16) { - self.steal_count += u64::from(by); - } - - pub(crate) fn incr_poll_count(&mut self) { - self.poll_count += 1; - } -} diff --git a/src/runtime/task/abort.rs b/src/runtime/task/abort.rs new file mode 100644 index 0000000..6edca10 --- /dev/null +++ b/src/runtime/task/abort.rs @@ -0,0 +1,87 @@ +use crate::runtime::task::{Header, RawTask}; +use std::fmt; +use std::panic::{RefUnwindSafe, UnwindSafe}; + +/// An owned permission to abort a spawned task, without awaiting its completion. +/// +/// Unlike a [`JoinHandle`], an `AbortHandle` does *not* represent the +/// permission to await the task's completion, only to terminate it. +/// +/// The task may be aborted by calling the [`AbortHandle::abort`] method. +/// Dropping an `AbortHandle` releases the permission to terminate the task +/// --- it does *not* abort the task. +/// +/// [`JoinHandle`]: crate::task::JoinHandle +#[cfg_attr(docsrs, doc(cfg(feature = "rt")))] +pub struct AbortHandle { + raw: RawTask, +} + +impl AbortHandle { + pub(super) fn new(raw: RawTask) -> Self { + Self { raw } + } + + /// Abort the task associated with the handle. + /// + /// Awaiting a cancelled task might complete as usual if the task was + /// already completed at the time it was cancelled, but most likely it + /// will fail with a [cancelled] `JoinError`. + /// + /// If the task was already cancelled, such as by [`JoinHandle::abort`], + /// this method will do nothing. + /// + /// [cancelled]: method@super::error::JoinError::is_cancelled + /// [`JoinHandle::abort`]: method@super::JoinHandle::abort + pub fn abort(&self) { + self.raw.remote_abort(); + } + + /// Checks if the task associated with this `AbortHandle` has finished. + /// + /// Please note that this method can return `false` even if `abort` has been + /// called on the task. This is because the cancellation process may take + /// some time, and this method does not return `true` until it has + /// completed. + pub fn is_finished(&self) -> bool { + let state = self.raw.state().load(); + state.is_complete() + } + + /// Returns a [task ID] that uniquely identifies this task relative to other + /// currently spawned tasks. + /// + /// **Note**: This is an [unstable API][unstable]. The public API of this type + /// may break in 1.x releases. See [the documentation on unstable + /// features][unstable] for details. + /// + /// [task ID]: crate::task::Id + /// [unstable]: crate#unstable-features + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn id(&self) -> super::Id { + // Safety: The header pointer is valid. + unsafe { Header::get_id(self.raw.header_ptr()) } + } +} + +unsafe impl Send for AbortHandle {} +unsafe impl Sync for AbortHandle {} + +impl UnwindSafe for AbortHandle {} +impl RefUnwindSafe for AbortHandle {} + +impl fmt::Debug for AbortHandle { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + // Safety: The header pointer is valid. + let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) }; + let id = unsafe { id_ptr.as_ref() }; + fmt.debug_struct("AbortHandle").field("id", id).finish() + } +} + +impl Drop for AbortHandle { + fn drop(&mut self) { + self.raw.drop_abort_handle(); + } +} diff --git a/src/runtime/task/core.rs b/src/runtime/task/core.rs index 776e834..bcccc69 100644 --- a/src/runtime/task/core.rs +++ b/src/runtime/task/core.rs @@ -11,9 +11,10 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; +use crate::runtime::context; use crate::runtime::task::raw::{self, Vtable}; use crate::runtime::task::state::State; -use crate::runtime::task::Schedule; +use crate::runtime::task::{Id, Schedule}; use crate::util::linked_list; use std::pin::Pin; @@ -24,6 +25,9 @@ use std::task::{Context, Poll, Waker}; /// /// It is critical for `Header` to be the first field as the task structure will /// be referenced by both *mut Cell and *mut Header. +/// +/// Any changes to the layout of this struct _must_ also be reflected in the +/// const fns in raw.rs. #[repr(C)] pub(super) struct Cell<T: Future, S> { /// Hot task state data @@ -43,10 +47,17 @@ pub(super) struct CoreStage<T: Future> { /// The core of the task. /// /// Holds the future or output, depending on the stage of execution. +/// +/// Any changes to the layout of this struct _must_ also be reflected in the +/// const fns in raw.rs. +#[repr(C)] pub(super) struct Core<T: Future, S> { /// Scheduler used to drive this future. pub(super) scheduler: S, + /// The task's ID, used for populating `JoinError`s. + pub(super) task_id: Id, + /// Either the future or the output. pub(super) stage: CoreStage<T>, } @@ -57,8 +68,6 @@ pub(crate) struct Header { /// Task state. pub(super) state: State, - pub(super) owned: UnsafeCell<linked_list::Pointers<Header>>, - /// Pointer to next task, used with the injection queue. pub(super) queue_next: UnsafeCell<Option<NonNull<Header>>>, @@ -80,18 +89,29 @@ pub(crate) struct Header { /// The tracing ID for this instrumented task. #[cfg(all(tokio_unstable, feature = "tracing"))] - pub(super) id: Option<tracing::Id>, + pub(super) tracing_id: Option<tracing::Id>, } unsafe impl Send for Header {} unsafe impl Sync for Header {} -/// Cold data is stored after the future. +/// Cold data is stored after the future. Data is considered cold if it is only +/// used during creation or shutdown of the task. pub(super) struct Trailer { + /// Pointers for the linked list in the `OwnedTasks` that owns this task. + pub(super) owned: linked_list::Pointers<Header>, /// Consumer task waiting on completion of this task. pub(super) waker: UnsafeCell<Option<Waker>>, } +generate_addr_of_methods! { + impl<> Trailer { + pub(super) unsafe fn addr_of_owned(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Header>> { + &self.owned + } + } +} + /// Either the future or the output. pub(super) enum Stage<T: Future> { Running(T), @@ -102,29 +122,48 @@ pub(super) enum Stage<T: Future> { impl<T: Future, S: Schedule> Cell<T, S> { /// Allocates a new task cell, containing the header, trailer, and core /// structures. - pub(super) fn new(future: T, scheduler: S, state: State) -> Box<Cell<T, S>> { + pub(super) fn new(future: T, scheduler: S, state: State, task_id: Id) -> Box<Cell<T, S>> { #[cfg(all(tokio_unstable, feature = "tracing"))] - let id = future.id(); - Box::new(Cell { + let tracing_id = future.id(); + let result = Box::new(Cell { header: Header { state, - owned: UnsafeCell::new(linked_list::Pointers::new()), queue_next: UnsafeCell::new(None), vtable: raw::vtable::<T, S>(), owner_id: UnsafeCell::new(0), #[cfg(all(tokio_unstable, feature = "tracing"))] - id, + tracing_id, }, core: Core { scheduler, stage: CoreStage { stage: UnsafeCell::new(Stage::Running(future)), }, + task_id, }, trailer: Trailer { waker: UnsafeCell::new(None), + owned: linked_list::Pointers::new(), }, - }) + }); + + #[cfg(debug_assertions)] + { + let trailer_addr = (&result.trailer) as *const Trailer as usize; + let trailer_ptr = unsafe { Header::get_trailer(NonNull::from(&result.header)) }; + assert_eq!(trailer_addr, trailer_ptr.as_ptr() as usize); + + let scheduler_addr = (&result.core.scheduler) as *const S as usize; + let scheduler_ptr = + unsafe { Header::get_scheduler::<S>(NonNull::from(&result.header)) }; + assert_eq!(scheduler_addr, scheduler_ptr.as_ptr() as usize); + + let id_addr = (&result.core.task_id) as *const Id as usize; + let id_ptr = unsafe { Header::get_id_ptr(NonNull::from(&result.header)) }; + assert_eq!(id_addr, id_ptr.as_ptr() as usize); + } + + result } } @@ -132,7 +171,29 @@ impl<T: Future> CoreStage<T> { pub(super) fn with_mut<R>(&self, f: impl FnOnce(*mut Stage<T>) -> R) -> R { self.stage.with_mut(f) } +} +/// Set and clear the task id in the context when the future is executed or +/// dropped, or when the output produced by the future is dropped. +pub(crate) struct TaskIdGuard { + parent_task_id: Option<Id>, +} + +impl TaskIdGuard { + fn enter(id: Id) -> Self { + TaskIdGuard { + parent_task_id: context::set_current_task_id(Some(id)), + } + } +} + +impl Drop for TaskIdGuard { + fn drop(&mut self) { + context::set_current_task_id(self.parent_task_id); + } +} + +impl<T: Future, S: Schedule> Core<T, S> { /// Polls the future. /// /// # Safety @@ -148,7 +209,7 @@ impl<T: Future> CoreStage<T> { /// heap. pub(super) fn poll(&self, mut cx: Context<'_>) -> Poll<T::Output> { let res = { - self.stage.with_mut(|ptr| { + self.stage.stage.with_mut(|ptr| { // Safety: The caller ensures mutual exclusion to the field. let future = match unsafe { &mut *ptr } { Stage::Running(future) => future, @@ -158,6 +219,7 @@ impl<T: Future> CoreStage<T> { // Safety: The caller ensures the future is pinned. let future = unsafe { Pin::new_unchecked(future) }; + let _guard = TaskIdGuard::enter(self.task_id); future.poll(&mut cx) }) }; @@ -201,7 +263,7 @@ impl<T: Future> CoreStage<T> { pub(super) fn take_output(&self) -> super::Result<T::Output> { use std::mem; - self.stage.with_mut(|ptr| { + self.stage.stage.with_mut(|ptr| { // Safety:: the caller ensures mutual exclusion to the field. match mem::replace(unsafe { &mut *ptr }, Stage::Consumed) { Stage::Finished(output) => output, @@ -211,7 +273,8 @@ impl<T: Future> CoreStage<T> { } unsafe fn set_stage(&self, stage: Stage<T>) { - self.stage.with_mut(|ptr| *ptr = stage) + let _guard = TaskIdGuard::enter(self.task_id); + self.stage.stage.with_mut(|ptr| *ptr = stage) } } @@ -236,6 +299,62 @@ impl Header { // the safety requirements on `set_owner_id`. unsafe { self.owner_id.with(|ptr| *ptr) } } + + /// Gets a pointer to the `Trailer` of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + pub(super) unsafe fn get_trailer(me: NonNull<Header>) -> NonNull<Trailer> { + let offset = me.as_ref().vtable.trailer_offset; + let trailer = me.as_ptr().cast::<u8>().add(offset).cast::<Trailer>(); + NonNull::new_unchecked(trailer) + } + + /// Gets a pointer to the scheduler of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + /// + /// The generic type S must be set to the correct scheduler type for this + /// task. + pub(super) unsafe fn get_scheduler<S>(me: NonNull<Header>) -> NonNull<S> { + let offset = me.as_ref().vtable.scheduler_offset; + let scheduler = me.as_ptr().cast::<u8>().add(offset).cast::<S>(); + NonNull::new_unchecked(scheduler) + } + + /// Gets a pointer to the id of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + pub(super) unsafe fn get_id_ptr(me: NonNull<Header>) -> NonNull<Id> { + let offset = me.as_ref().vtable.id_offset; + let id = me.as_ptr().cast::<u8>().add(offset).cast::<Id>(); + NonNull::new_unchecked(id) + } + + /// Gets the id of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + pub(super) unsafe fn get_id(me: NonNull<Header>) -> Id { + let ptr = Header::get_id_ptr(me).as_ptr(); + *ptr + } + + /// Gets the tracing id of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) unsafe fn get_tracing_id(me: &NonNull<Header>) -> Option<&tracing::Id> { + me.as_ref().tracing_id.as_ref() + } } impl Trailer { diff --git a/src/runtime/task/error.rs b/src/runtime/task/error.rs index 1a8129b..f7ead77 100644 --- a/src/runtime/task/error.rs +++ b/src/runtime/task/error.rs @@ -2,12 +2,13 @@ use std::any::Any; use std::fmt; use std::io; +use super::Id; use crate::util::SyncWrapper; - cfg_rt! { /// Task failed to execute to completion. pub struct JoinError { repr: Repr, + id: Id, } } @@ -17,15 +18,17 @@ enum Repr { } impl JoinError { - pub(crate) fn cancelled() -> JoinError { + pub(crate) fn cancelled(id: Id) -> JoinError { JoinError { repr: Repr::Cancelled, + id, } } - pub(crate) fn panic(err: Box<dyn Any + Send + 'static>) -> JoinError { + pub(crate) fn panic(id: Id, err: Box<dyn Any + Send + 'static>) -> JoinError { JoinError { repr: Repr::Panic(SyncWrapper::new(err)), + id, } } @@ -79,6 +82,7 @@ impl JoinError { /// } /// } /// ``` + #[track_caller] pub fn into_panic(self) -> Box<dyn Any + Send + 'static> { self.try_into_panic() .expect("`JoinError` reason is not a panic.") @@ -111,13 +115,28 @@ impl JoinError { _ => Err(self), } } + + /// Returns a [task ID] that identifies the task which errored relative to + /// other currently spawned tasks. + /// + /// **Note**: This is an [unstable API][unstable]. The public API of this type + /// may break in 1.x releases. See [the documentation on unstable + /// features][unstable] for details. + /// + /// [task ID]: crate::task::Id + /// [unstable]: crate#unstable-features + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn id(&self) -> Id { + self.id + } } impl fmt::Display for JoinError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.repr { - Repr::Cancelled => write!(fmt, "cancelled"), - Repr::Panic(_) => write!(fmt, "panic"), + Repr::Cancelled => write!(fmt, "task {} was cancelled", self.id), + Repr::Panic(_) => write!(fmt, "task {} panicked", self.id), } } } @@ -125,8 +144,8 @@ impl fmt::Display for JoinError { impl fmt::Debug for JoinError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.repr { - Repr::Cancelled => write!(fmt, "JoinError::Cancelled"), - Repr::Panic(_) => write!(fmt, "JoinError::Panic(...)"), + Repr::Cancelled => write!(fmt, "JoinError::Cancelled({:?})", self.id), + Repr::Panic(_) => write!(fmt, "JoinError::Panic({:?}, ...)", self.id), } } } diff --git a/src/runtime/task/harness.rs b/src/runtime/task/harness.rs index 0996e52..8e3c3d1 100644 --- a/src/runtime/task/harness.rs +++ b/src/runtime/task/harness.rs @@ -1,8 +1,8 @@ use crate::future::Future; -use crate::runtime::task::core::{Cell, Core, CoreStage, Header, Trailer}; -use crate::runtime::task::state::Snapshot; +use crate::runtime::task::core::{Cell, Core, Header, Trailer}; +use crate::runtime::task::state::{Snapshot, State}; use crate::runtime::task::waker::waker_ref; -use crate::runtime::task::{JoinError, Notified, Schedule, Task}; +use crate::runtime::task::{JoinError, Notified, RawTask, Schedule, Task}; use std::mem; use std::mem::ManuallyDrop; @@ -26,8 +26,16 @@ where } } + fn header_ptr(&self) -> NonNull<Header> { + self.cell.cast() + } + fn header(&self) -> &Header { - unsafe { &self.cell.as_ref().header } + unsafe { &*self.header_ptr().as_ptr() } + } + + fn state(&self) -> &State { + &self.header().state } fn trailer(&self) -> &Trailer { @@ -39,11 +47,102 @@ where } } +/// Task operations that can be implemented without being generic over the +/// scheduler or task. Only one version of these methods should exist in the +/// final binary. +impl RawTask { + pub(super) fn drop_reference(self) { + if self.state().ref_dec() { + self.dealloc(); + } + } + + /// This call consumes a ref-count and notifies the task. This will create a + /// new Notified and submit it if necessary. + /// + /// The caller does not need to hold a ref-count besides the one that was + /// passed to this call. + pub(super) fn wake_by_val(&self) { + use super::state::TransitionToNotifiedByVal; + + match self.state().transition_to_notified_by_val() { + TransitionToNotifiedByVal::Submit => { + // The caller has given us a ref-count, and the transition has + // created a new ref-count, so we now hold two. We turn the new + // ref-count Notified and pass it to the call to `schedule`. + // + // The old ref-count is retained for now to ensure that the task + // is not dropped during the call to `schedule` if the call + // drops the task it was given. + self.schedule(); + + // Now that we have completed the call to schedule, we can + // release our ref-count. + self.drop_reference(); + } + TransitionToNotifiedByVal::Dealloc => { + self.dealloc(); + } + TransitionToNotifiedByVal::DoNothing => {} + } + } + + /// This call notifies the task. It will not consume any ref-counts, but the + /// caller should hold a ref-count. This will create a new Notified and + /// submit it if necessary. + pub(super) fn wake_by_ref(&self) { + use super::state::TransitionToNotifiedByRef; + + match self.state().transition_to_notified_by_ref() { + TransitionToNotifiedByRef::Submit => { + // The transition above incremented the ref-count for a new task + // and the caller also holds a ref-count. The caller's ref-count + // ensures that the task is not destroyed even if the new task + // is dropped before `schedule` returns. + self.schedule(); + } + TransitionToNotifiedByRef::DoNothing => {} + } + } + + /// Remotely aborts the task. + /// + /// The caller should hold a ref-count, but we do not consume it. + /// + /// This is similar to `shutdown` except that it asks the runtime to perform + /// the shutdown. This is necessary to avoid the shutdown happening in the + /// wrong thread for non-Send tasks. + pub(super) fn remote_abort(&self) { + if self.state().transition_to_notified_and_cancel() { + // The transition has created a new ref-count, which we turn into + // a Notified and pass to the task. + // + // Since the caller holds a ref-count, the task cannot be destroyed + // before the call to `schedule` returns even if the call drops the + // `Notified` internally. + self.schedule(); + } + } + + /// Try to set the waker notified when the task is complete. Returns true if + /// the task has already completed. If this call returns false, then the + /// waker will not be notified. + pub(super) fn try_set_join_waker(&self, waker: &Waker) -> bool { + can_read_output(self.header(), self.trailer(), waker) + } +} + impl<T, S> Harness<T, S> where T: Future, S: Schedule, { + pub(super) fn drop_reference(self) { + if self.state().ref_dec() { + self.dealloc(); + } + } + /// Polls the inner future. A ref-count is consumed. /// /// All necessary state checks and transitions are performed. @@ -91,32 +190,32 @@ where fn poll_inner(&self) -> PollFuture { use super::state::{TransitionToIdle, TransitionToRunning}; - match self.header().state.transition_to_running() { + match self.state().transition_to_running() { TransitionToRunning::Success => { - let waker_ref = waker_ref::<T, S>(self.header()); - let cx = Context::from_waker(&*waker_ref); - let res = poll_future(&self.core().stage, cx); + let header_ptr = self.header_ptr(); + let waker_ref = waker_ref::<T, S>(&header_ptr); + let cx = Context::from_waker(&waker_ref); + let res = poll_future(self.core(), cx); if res == Poll::Ready(()) { // The future completed. Move on to complete the task. return PollFuture::Complete; } - match self.header().state.transition_to_idle() { + match self.state().transition_to_idle() { TransitionToIdle::Ok => PollFuture::Done, TransitionToIdle::OkNotified => PollFuture::Notified, TransitionToIdle::OkDealloc => PollFuture::Dealloc, TransitionToIdle::Cancelled => { // The transition to idle failed because the task was // cancelled during the poll. - - cancel_task(&self.core().stage); + cancel_task(self.core()); PollFuture::Complete } } } TransitionToRunning::Cancelled => { - cancel_task(&self.core().stage); + cancel_task(self.core()); PollFuture::Complete } TransitionToRunning::Failed => PollFuture::Done, @@ -131,7 +230,7 @@ where /// there is nothing further to do. When the task completes running, it will /// notice the `CANCELLED` bit and finalize the task. pub(super) fn shutdown(self) { - if !self.header().state.transition_to_shutdown() { + if !self.state().transition_to_shutdown() { // The task is concurrently running. No further work needed. self.drop_reference(); return; @@ -139,7 +238,7 @@ where // By transitioning the lifecycle to `Running`, we have permission to // drop the future. - cancel_task(&self.core().stage); + cancel_task(self.core()); self.complete(); } @@ -150,6 +249,19 @@ where // Check causality self.core().stage.with_mut(drop); + // Safety: The caller of this method just transitioned our ref-count to + // zero, so it is our responsibility to release the allocation. + // + // We don't hold any references into the allocation at this point, but + // it is possible for another thread to still hold a `&State` into the + // allocation if that other thread has decremented its last ref-count, + // but has not yet returned from the relevant method on `State`. + // + // However, the `State` type consists of just an `AtomicUsize`, and an + // `AtomicUsize` wraps the entirety of its contents in an `UnsafeCell`. + // As explained in the documentation for `UnsafeCell`, such references + // are allowed to be dangling after their last use, even if the + // reference has not yet gone out of scope. unsafe { drop(Box::from_raw(self.cell.as_ptr())); } @@ -160,122 +272,30 @@ where /// Read the task output into `dst`. pub(super) fn try_read_output(self, dst: &mut Poll<super::Result<T::Output>>, waker: &Waker) { if can_read_output(self.header(), self.trailer(), waker) { - *dst = Poll::Ready(self.core().stage.take_output()); + *dst = Poll::Ready(self.core().take_output()); } } pub(super) fn drop_join_handle_slow(self) { - let mut maybe_panic = None; - // Try to unset `JOIN_INTEREST`. This must be done as a first step in // case the task concurrently completed. - if self.header().state.unset_join_interested().is_err() { + if self.state().unset_join_interested().is_err() { // It is our responsibility to drop the output. This is critical as // the task output may not be `Send` and as such must remain with // the scheduler or `JoinHandle`. i.e. if the output remains in the // task structure until the task is deallocated, it may be dropped // by a Waker on any arbitrary thread. - let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { - self.core().stage.drop_future_or_output(); + // + // Panics are delivered to the user via the `JoinHandle`. Given that + // they are dropping the `JoinHandle`, we assume they are not + // interested in the panic and swallow it. + let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| { + self.core().drop_future_or_output(); })); - - if let Err(panic) = panic { - maybe_panic = Some(panic); - } } // Drop the `JoinHandle` reference, possibly deallocating the task self.drop_reference(); - - if let Some(panic) = maybe_panic { - panic::resume_unwind(panic); - } - } - - /// Remotely aborts the task. - /// - /// The caller should hold a ref-count, but we do not consume it. - /// - /// This is similar to `shutdown` except that it asks the runtime to perform - /// the shutdown. This is necessary to avoid the shutdown happening in the - /// wrong thread for non-Send tasks. - pub(super) fn remote_abort(self) { - if self.header().state.transition_to_notified_and_cancel() { - // The transition has created a new ref-count, which we turn into - // a Notified and pass to the task. - // - // Since the caller holds a ref-count, the task cannot be destroyed - // before the call to `schedule` returns even if the call drops the - // `Notified` internally. - self.core() - .scheduler - .schedule(Notified(self.get_new_task())); - } - } - - // ===== waker behavior ===== - - /// This call consumes a ref-count and notifies the task. This will create a - /// new Notified and submit it if necessary. - /// - /// The caller does not need to hold a ref-count besides the one that was - /// passed to this call. - pub(super) fn wake_by_val(self) { - use super::state::TransitionToNotifiedByVal; - - match self.header().state.transition_to_notified_by_val() { - TransitionToNotifiedByVal::Submit => { - // The caller has given us a ref-count, and the transition has - // created a new ref-count, so we now hold two. We turn the new - // ref-count Notified and pass it to the call to `schedule`. - // - // The old ref-count is retained for now to ensure that the task - // is not dropped during the call to `schedule` if the call - // drops the task it was given. - self.core() - .scheduler - .schedule(Notified(self.get_new_task())); - - // Now that we have completed the call to schedule, we can - // release our ref-count. - self.drop_reference(); - } - TransitionToNotifiedByVal::Dealloc => { - self.dealloc(); - } - TransitionToNotifiedByVal::DoNothing => {} - } - } - - /// This call notifies the task. It will not consume any ref-counts, but the - /// caller should hold a ref-count. This will create a new Notified and - /// submit it if necessary. - pub(super) fn wake_by_ref(&self) { - use super::state::TransitionToNotifiedByRef; - - match self.header().state.transition_to_notified_by_ref() { - TransitionToNotifiedByRef::Submit => { - // The transition above incremented the ref-count for a new task - // and the caller also holds a ref-count. The caller's ref-count - // ensures that the task is not destroyed even if the new task - // is dropped before `schedule` returns. - self.core() - .scheduler - .schedule(Notified(self.get_new_task())); - } - TransitionToNotifiedByRef::DoNothing => {} - } - } - - pub(super) fn drop_reference(self) { - if self.header().state.ref_dec() { - self.dealloc(); - } - } - - #[cfg(all(tokio_unstable, feature = "tracing"))] - pub(super) fn id(&self) -> Option<&tracing::Id> { - self.header().id.as_ref() } // ====== internal ====== @@ -285,7 +305,7 @@ where // The future has completed and its output has been written to the task // stage. We transition from running to complete. - let snapshot = self.header().state.transition_to_complete(); + let snapshot = self.state().transition_to_complete(); // We catch panics here in case dropping the future or waking the // JoinHandle panics. @@ -294,10 +314,11 @@ where // The `JoinHandle` is not interested in the output of // this task. It is our responsibility to drop the // output. - self.core().stage.drop_future_or_output(); - } else if snapshot.has_join_waker() { - // Notify the join handle. The previous transition obtains the - // lock on the waker cell. + self.core().drop_future_or_output(); + } else if snapshot.is_join_waker_set() { + // Notify the waker. Reading the waker field is safe per rule 4 + // in task/mod.rs, since the JOIN_WAKER bit is set and the call + // to transition_to_complete() above set the COMPLETE bit. self.trailer().wake_join(); } })); @@ -305,7 +326,7 @@ where // The task has completed execution and will no longer be scheduled. let num_release = self.release(); - if self.header().state.transition_to_terminal(num_release) { + if self.state().transition_to_terminal(num_release) { self.dealloc(); } } @@ -347,36 +368,30 @@ fn can_read_output(header: &Header, trailer: &Trailer, waker: &Waker) -> bool { debug_assert!(snapshot.is_join_interested()); if !snapshot.is_complete() { - // The waker must be stored in the task struct. - let res = if snapshot.has_join_waker() { - // There already is a waker stored in the struct. If it matches - // the provided waker, then there is no further work to do. - // Otherwise, the waker must be swapped. - let will_wake = unsafe { - // Safety: when `JOIN_INTEREST` is set, only `JOIN_HANDLE` - // may mutate the `waker` field. - trailer.will_wake(waker) - }; - - if will_wake { - // The task is not complete **and** the waker is up to date, - // there is nothing further that needs to be done. + // If the task is not complete, try storing the provided waker in the + // task's waker field. + + let res = if snapshot.is_join_waker_set() { + // If JOIN_WAKER is set, then JoinHandle has previously stored a + // waker in the waker field per step (iii) of rule 5 in task/mod.rs. + + // Optimization: if the stored waker and the provided waker wake the + // same task, then return without touching the waker field. (Reading + // the waker field below is safe per rule 3 in task/mod.rs.) + if unsafe { trailer.will_wake(waker) } { return false; } - // Unset the `JOIN_WAKER` to gain mutable access to the `waker` - // field then update the field with the new join worker. - // - // This requires two atomic operations, unsetting the bit and - // then resetting it. If the task transitions to complete - // concurrently to either one of those operations, then setting - // the join waker fails and we proceed to reading the task - // output. + // Otherwise swap the stored waker with the provided waker by + // following the rule 5 in task/mod.rs. header .state .unset_waker() .and_then(|snapshot| set_join_waker(header, trailer, waker.clone(), snapshot)) } else { + // If JOIN_WAKER is unset, then JoinHandle has mutable access to the + // waker field per rule 2 in task/mod.rs; therefore, skip step (i) + // of rule 5 and try to store the provided waker in the waker field. set_join_waker(header, trailer, waker.clone(), snapshot) }; @@ -397,7 +412,7 @@ fn set_join_waker( snapshot: Snapshot, ) -> Result<Snapshot, Snapshot> { assert!(snapshot.is_join_interested()); - assert!(!snapshot.has_join_waker()); + assert!(!snapshot.is_join_waker_set()); // Safety: Only the `JoinHandle` may set the `waker` field. When // `JOIN_INTEREST` is **not** set, nothing else will touch the field. @@ -426,31 +441,31 @@ enum PollFuture { } /// Cancels the task and store the appropriate error in the stage field. -fn cancel_task<T: Future>(stage: &CoreStage<T>) { +fn cancel_task<T: Future, S: Schedule>(core: &Core<T, S>) { // Drop the future from a panic guard. let res = panic::catch_unwind(panic::AssertUnwindSafe(|| { - stage.drop_future_or_output(); + core.drop_future_or_output(); })); match res { Ok(()) => { - stage.store_output(Err(JoinError::cancelled())); + core.store_output(Err(JoinError::cancelled(core.task_id))); } Err(panic) => { - stage.store_output(Err(JoinError::panic(panic))); + core.store_output(Err(JoinError::panic(core.task_id, panic))); } } } /// Polls the future. If the future completes, the output is written to the /// stage field. -fn poll_future<T: Future>(core: &CoreStage<T>, cx: Context<'_>) -> Poll<()> { +fn poll_future<T: Future, S: Schedule>(core: &Core<T, S>, cx: Context<'_>) -> Poll<()> { // Poll the future. let output = panic::catch_unwind(panic::AssertUnwindSafe(|| { - struct Guard<'a, T: Future> { - core: &'a CoreStage<T>, + struct Guard<'a, T: Future, S: Schedule> { + core: &'a Core<T, S>, } - impl<'a, T: Future> Drop for Guard<'a, T> { + impl<'a, T: Future, S: Schedule> Drop for Guard<'a, T, S> { fn drop(&mut self) { // If the future panics on poll, we drop it inside the panic // guard. @@ -467,13 +482,20 @@ fn poll_future<T: Future>(core: &CoreStage<T>, cx: Context<'_>) -> Poll<()> { let output = match output { Ok(Poll::Pending) => return Poll::Pending, Ok(Poll::Ready(output)) => Ok(output), - Err(panic) => Err(JoinError::panic(panic)), + Err(panic) => { + core.scheduler.unhandled_panic(); + Err(JoinError::panic(core.task_id, panic)) + } }; // Catch and ignore panics if the future panics on drop. - let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| { + let res = panic::catch_unwind(panic::AssertUnwindSafe(|| { core.store_output(output); })); + if res.is_err() { + core.scheduler.unhandled_panic(); + } + Poll::Ready(()) } diff --git a/src/runtime/task/id.rs b/src/runtime/task/id.rs new file mode 100644 index 0000000..2b0d95c --- /dev/null +++ b/src/runtime/task/id.rs @@ -0,0 +1,87 @@ +use crate::runtime::context; + +use std::fmt; + +/// An opaque ID that uniquely identifies a task relative to all other currently +/// running tasks. +/// +/// # Notes +/// +/// - Task IDs are unique relative to other *currently running* tasks. When a +/// task completes, the same ID may be used for another task. +/// - Task IDs are *not* sequential, and do not indicate the order in which +/// tasks are spawned, what runtime a task is spawned on, or any other data. +/// - The task ID of the currently running task can be obtained from inside the +/// task via the [`task::try_id()`](crate::task::try_id()) and +/// [`task::id()`](crate::task::id()) functions and from outside the task via +/// the [`JoinHandle::id()`](crate::task::JoinHandle::id()) function. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [unstable]: crate#unstable-features +#[cfg_attr(docsrs, doc(cfg(all(feature = "rt", tokio_unstable))))] +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] +pub struct Id(u64); + +/// Returns the [`Id`] of the currently running task. +/// +/// # Panics +/// +/// This function panics if called from outside a task. Please note that calls +/// to `block_on` do not have task IDs, so the method will panic if called from +/// within a call to `block_on`. For a version of this function that doesn't +/// panic, see [`task::try_id()`](crate::runtime::task::try_id()). +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [task ID]: crate::task::Id +/// [unstable]: crate#unstable-features +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +#[track_caller] +pub fn id() -> Id { + context::current_task_id().expect("Can't get a task id when not inside a task") +} + +/// Returns the [`Id`] of the currently running task, or `None` if called outside +/// of a task. +/// +/// This function is similar to [`task::id()`](crate::runtime::task::id()), except +/// that it returns `None` rather than panicking if called outside of a task +/// context. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [task ID]: crate::task::Id +/// [unstable]: crate#unstable-features +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +#[track_caller] +pub fn try_id() -> Option<Id> { + context::current_task_id() +} + +impl fmt::Display for Id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Id { + pub(crate) fn next() -> Self { + use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64}; + + static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1); + + Self(NEXT_ID.fetch_add(1, Relaxed)) + } + + pub(crate) fn as_u64(&self) -> u64 { + self.0 + } +} diff --git a/src/runtime/task/join.rs b/src/runtime/task/join.rs index 0abbff2..11c4b9b 100644 --- a/src/runtime/task/join.rs +++ b/src/runtime/task/join.rs @@ -1,16 +1,19 @@ -use crate::runtime::task::RawTask; +use crate::runtime::task::{Header, RawTask}; use std::fmt; use std::future::Future; use std::marker::PhantomData; +use std::panic::{RefUnwindSafe, UnwindSafe}; use std::pin::Pin; -use std::task::{Context, Poll}; +use std::task::{Context, Poll, Waker}; cfg_rt! { /// An owned permission to join on a task (await its termination). /// - /// This can be thought of as the equivalent of [`std::thread::JoinHandle`] for - /// a task rather than a thread. + /// This can be thought of as the equivalent of [`std::thread::JoinHandle`] + /// for a Tokio task rather than a thread. Note that the background task + /// associated with this `JoinHandle` started running immediately when you + /// called spawn, even if you have not yet awaited the `JoinHandle`. /// /// A `JoinHandle` *detaches* the associated task when it is dropped, which /// means that there is no longer any handle to the task, and no way to `join` @@ -19,6 +22,15 @@ cfg_rt! { /// This `struct` is created by the [`task::spawn`] and [`task::spawn_blocking`] /// functions. /// + /// # Cancel safety + /// + /// The `&mut JoinHandle<T>` type is cancel safe. If it is used as the event + /// in a `tokio::select!` statement and some other branch completes first, + /// then it is guaranteed that the output of the task is not lost. + /// + /// If a `JoinHandle` is dropped, then the task continues running in the + /// background and its return value is lost. + /// /// # Examples /// /// Creation from [`task::spawn`]: @@ -142,7 +154,7 @@ cfg_rt! { /// [`std::thread::JoinHandle`]: std::thread::JoinHandle /// [`JoinError`]: crate::task::JoinError pub struct JoinHandle<T> { - raw: Option<RawTask>, + raw: RawTask, _p: PhantomData<T>, } } @@ -150,10 +162,13 @@ cfg_rt! { unsafe impl<T: Send> Send for JoinHandle<T> {} unsafe impl<T: Send> Sync for JoinHandle<T> {} +impl<T> UnwindSafe for JoinHandle<T> {} +impl<T> RefUnwindSafe for JoinHandle<T> {} + impl<T> JoinHandle<T> { pub(super) fn new(raw: RawTask) -> JoinHandle<T> { JoinHandle { - raw: Some(raw), + raw, _p: PhantomData, } } @@ -192,10 +207,71 @@ impl<T> JoinHandle<T> { /// ``` /// [cancelled]: method@super::error::JoinError::is_cancelled pub fn abort(&self) { - if let Some(raw) = self.raw { - raw.remote_abort(); + self.raw.remote_abort(); + } + + /// Checks if the task associated with this `JoinHandle` has finished. + /// + /// Please note that this method can return `false` even if [`abort`] has been + /// called on the task. This is because the cancellation process may take + /// some time, and this method does not return `true` until it has + /// completed. + /// + /// ```rust + /// use tokio::time; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() { + /// # time::pause(); + /// let handle1 = tokio::spawn(async { + /// // do some stuff here + /// }); + /// let handle2 = tokio::spawn(async { + /// // do some other stuff here + /// time::sleep(time::Duration::from_secs(10)).await; + /// }); + /// // Wait for the task to finish + /// handle2.abort(); + /// time::sleep(time::Duration::from_secs(1)).await; + /// assert!(handle1.is_finished()); + /// assert!(handle2.is_finished()); + /// # } + /// ``` + /// [`abort`]: method@JoinHandle::abort + pub fn is_finished(&self) -> bool { + let state = self.raw.header().state.load(); + state.is_complete() + } + + /// Set the waker that is notified when the task completes. + pub(crate) fn set_join_waker(&mut self, waker: &Waker) { + if self.raw.try_set_join_waker(waker) { + // In this case the task has already completed. We wake the waker immediately. + waker.wake_by_ref(); } } + + /// Returns a new `AbortHandle` that can be used to remotely abort this task. + pub(crate) fn abort_handle(&self) -> super::AbortHandle { + self.raw.ref_inc(); + super::AbortHandle::new(self.raw) + } + + /// Returns a [task ID] that uniquely identifies this task relative to other + /// currently spawned tasks. + /// + /// **Note**: This is an [unstable API][unstable]. The public API of this type + /// may break in 1.x releases. See [the documentation on unstable + /// features][unstable] for details. + /// + /// [task ID]: crate::task::Id + /// [unstable]: crate#unstable-features + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn id(&self) -> super::Id { + // Safety: The header pointer is valid. + unsafe { Header::get_id(self.raw.header_ptr()) } + } } impl<T> Unpin for JoinHandle<T> {} @@ -207,14 +283,7 @@ impl<T> Future for JoinHandle<T> { let mut ret = Poll::Pending; // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); - - // Raw should always be set. If it is not, this is due to polling after - // completion - let raw = self - .raw - .as_ref() - .expect("polling after `JoinHandle` already completed"); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); // Try to read the task output. If the task is not yet complete, the // waker is stored and is notified once the task does complete. @@ -228,7 +297,8 @@ impl<T> Future for JoinHandle<T> { // // The type of `T` must match the task's output type. unsafe { - raw.try_read_output(&mut ret as *mut _ as *mut (), cx.waker()); + self.raw + .try_read_output(&mut ret as *mut _ as *mut (), cx.waker()); } if ret.is_ready() { @@ -241,13 +311,11 @@ impl<T> Future for JoinHandle<T> { impl<T> Drop for JoinHandle<T> { fn drop(&mut self) { - if let Some(raw) = self.raw.take() { - if raw.header().state.drop_join_handle_fast().is_ok() { - return; - } - - raw.drop_join_handle_slow(); + if self.raw.state().drop_join_handle_fast().is_ok() { + return; } + + self.raw.drop_join_handle_slow(); } } @@ -256,6 +324,9 @@ where T: fmt::Debug, { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("JoinHandle").finish() + // Safety: The header pointer is valid. + let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) }; + let id = unsafe { id_ptr.as_ref() }; + fmt.debug_struct("JoinHandle").field("id", id).finish() } } diff --git a/src/runtime/task/list.rs b/src/runtime/task/list.rs index 7758f8d..159c13e 100644 --- a/src/runtime/task/list.rs +++ b/src/runtime/task/list.rs @@ -84,13 +84,14 @@ impl<S: 'static> OwnedTasks<S> { &self, task: T, scheduler: S, + id: super::Id, ) -> (JoinHandle<T::Output>, Option<Notified<S>>) where S: Schedule, T: Future + Send + 'static, T::Output: Send + 'static, { - let (task, notified, join) = super::new_task(task, scheduler); + let (task, notified, join) = super::new_task(task, scheduler, id); unsafe { // safety: We just created the task, so we have exclusive access @@ -163,7 +164,7 @@ impl<S: 'static> OwnedTasks<S> { // safety: We just checked that the provided task is not in some other // linked list. - unsafe { self.inner.lock().list.remove(task.header().into()) } + unsafe { self.inner.lock().list.remove(task.header_ptr()) } } pub(crate) fn is_empty(&self) -> bool { @@ -187,13 +188,14 @@ impl<S: 'static> LocalOwnedTasks<S> { &self, task: T, scheduler: S, + id: super::Id, ) -> (JoinHandle<T::Output>, Option<Notified<S>>) where S: Schedule, T: Future + 'static, T::Output: 'static, { - let (task, notified, join) = super::new_task(task, scheduler); + let (task, notified, join) = super::new_task(task, scheduler, id); unsafe { // safety: We just created the task, so we have exclusive access @@ -238,7 +240,7 @@ impl<S: 'static> LocalOwnedTasks<S> { self.with_inner(|inner| // safety: We just checked that the provided task is not in some // other linked list. - unsafe { inner.list.remove(task.header().into()) }) + unsafe { inner.list.remove(task.header_ptr()) }) } /// Asserts that the given task is owned by this LocalOwnedTasks and convert diff --git a/src/runtime/task/mod.rs b/src/runtime/task/mod.rs index 1f18209..55131ac 100644 --- a/src/runtime/task/mod.rs +++ b/src/runtime/task/mod.rs @@ -25,7 +25,7 @@ //! //! The task uses a reference count to keep track of how many active references //! exist. The Unowned reference type takes up two ref-counts. All other -//! reference types take pu a single ref-count. +//! reference types take up a single ref-count. //! //! Besides the waker type, each task has at most one of each reference type. //! @@ -47,7 +47,8 @@ //! //! * JOIN_INTEREST - Is set to one if there exists a JoinHandle. //! -//! * JOIN_WAKER - Is set to one if the JoinHandle has set a waker. +//! * JOIN_WAKER - Acts as an access control bit for the join handle waker. The +//! protocol for its usage is described below. //! //! The rest of the bits are used for the ref-count. //! @@ -71,10 +72,38 @@ //! a lock for the stage field, and it can be accessed only by the thread //! that set RUNNING to one. //! -//! * If JOIN_WAKER is zero, then the JoinHandle has exclusive access to the -//! join handle waker. If JOIN_WAKER and COMPLETE are both one, then the -//! thread that set COMPLETE to one has exclusive access to the join handle -//! waker. +//! * The waker field may be concurrently accessed by different threads: in one +//! thread the runtime may complete a task and *read* the waker field to +//! invoke the waker, and in another thread the task's JoinHandle may be +//! polled, and if the task hasn't yet completed, the JoinHandle may *write* +//! a waker to the waker field. The JOIN_WAKER bit ensures safe access by +//! multiple threads to the waker field using the following rules: +//! +//! 1. JOIN_WAKER is initialized to zero. +//! +//! 2. If JOIN_WAKER is zero, then the JoinHandle has exclusive (mutable) +//! access to the waker field. +//! +//! 3. If JOIN_WAKER is one, then the JoinHandle has shared (read-only) +//! access to the waker field. +//! +//! 4. If JOIN_WAKER is one and COMPLETE is one, then the runtime has shared +//! (read-only) access to the waker field. +//! +//! 5. If the JoinHandle needs to write to the waker field, then the +//! JoinHandle needs to (i) successfully set JOIN_WAKER to zero if it is +//! not already zero to gain exclusive access to the waker field per rule +//! 2, (ii) write a waker, and (iii) successfully set JOIN_WAKER to one. +//! +//! 6. The JoinHandle can change JOIN_WAKER only if COMPLETE is zero (i.e. +//! the task hasn't yet completed). +//! +//! Rule 6 implies that the steps (i) or (iii) of rule 5 may fail due to a +//! race. If step (i) fails, then the attempt to write a waker is aborted. If +//! step (iii) fails because COMPLETE is set to one by another thread after +//! step (i), then the waker field is cleared. Once COMPLETE is one (i.e. +//! task has completed), the JoinHandle will not modify JOIN_WAKER. After the +//! runtime sets COMPLETE to one, it invokes the waker if there is one. //! //! All other fields are immutable and can be accessed immutably without //! synchronization by anyone. @@ -121,7 +150,7 @@ //! 1. The output is created on the thread that the future was polled on. Since //! only non-Send futures can have non-Send output, the future was polled on //! the thread that the future was spawned from. -//! 2. Since JoinHandle<Output> is not Send if Output is not Send, the +//! 2. Since `JoinHandle<Output>` is not Send if Output is not Send, the //! JoinHandle is also on the thread that the future was spawned from. //! 3. Thus, the JoinHandle will not move the output across threads when it //! takes or drops the output. @@ -135,24 +164,36 @@ //! poll call will notice it when the poll finishes, and the task is cancelled //! at that point. +// Some task infrastructure is here to support `JoinSet`, which is currently +// unstable. This should be removed once `JoinSet` is stabilized. +#![cfg_attr(not(tokio_unstable), allow(dead_code))] + mod core; use self::core::Cell; use self::core::Header; mod error; -#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411 pub use self::error::JoinError; mod harness; use self::harness::Harness; +mod id; +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +pub use id::{id, try_id, Id}; + cfg_rt_multi_thread! { mod inject; pub(super) use self::inject::Inject; } +#[cfg(feature = "rt")] +mod abort; mod join; -#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411 + +#[cfg(feature = "rt")] +pub use self::abort::AbortHandle; + pub use self::join::JoinHandle; mod list; @@ -230,6 +271,11 @@ pub(crate) trait Schedule: Sync + Sized + 'static { fn yield_now(&self, task: Notified<Self>) { self.schedule(task); } + + /// Polling the task resulted in a panic. Should the runtime shutdown? + fn unhandled_panic(&self) { + // By default, do nothing. This maintains the 1.0 behavior. + } } cfg_rt! { @@ -239,14 +285,15 @@ cfg_rt! { /// notification. fn new_task<T, S>( task: T, - scheduler: S + scheduler: S, + id: Id, ) -> (Task<S>, Notified<S>, JoinHandle<T::Output>) where S: Schedule, T: Future + 'static, T::Output: 'static, { - let raw = RawTask::new::<T, S>(task, scheduler); + let raw = RawTask::new::<T, S>(task, scheduler, id); let task = Task { raw, _p: PhantomData, @@ -264,13 +311,13 @@ cfg_rt! { /// only when the task is not going to be stored in an `OwnedTasks` list. /// /// Currently only blocking tasks use this method. - pub(crate) fn unowned<T, S>(task: T, scheduler: S) -> (UnownedTask<S>, JoinHandle<T::Output>) + pub(crate) fn unowned<T, S>(task: T, scheduler: S, id: Id) -> (UnownedTask<S>, JoinHandle<T::Output>) where S: Schedule, T: Send + Future + 'static, T::Output: Send + 'static, { - let (task, notified, join) = new_task(task, scheduler); + let (task, notified, join) = new_task(task, scheduler, id); // This transfers the ref-count of task and notified into an UnownedTask. // This is valid because an UnownedTask holds two ref-counts. @@ -296,6 +343,10 @@ impl<S: 'static> Task<S> { fn header(&self) -> &Header { self.raw.header() } + + fn header_ptr(&self) -> NonNull<Header> { + self.raw.header_ptr() + } } impl<S: 'static> Notified<S> { @@ -313,7 +364,7 @@ cfg_rt_multi_thread! { impl<S: 'static> Task<S> { fn into_raw(self) -> NonNull<Header> { - let ret = self.header().into(); + let ret = self.raw.header_ptr(); mem::forget(self); ret } @@ -327,7 +378,7 @@ cfg_rt_multi_thread! { } impl<S: Schedule> Task<S> { - /// Pre-emptively cancels the task as part of the shutdown process. + /// Preemptively cancels the task as part of the shutdown process. pub(crate) fn shutdown(self) { let raw = self.raw; mem::forget(self); @@ -347,6 +398,7 @@ impl<S: Schedule> LocalNotified<S> { impl<S: Schedule> UnownedTask<S> { // Used in test of the inject queue. #[cfg(test)] + #[cfg_attr(tokio_wasm, allow(dead_code))] pub(super) fn into_notified(self) -> Notified<S> { Notified(self.into_task()) } @@ -426,7 +478,7 @@ unsafe impl<S> linked_list::Link for Task<S> { type Target = Header; fn as_raw(handle: &Task<S>) -> NonNull<Header> { - handle.header().into() + handle.raw.header_ptr() } unsafe fn from_raw(ptr: NonNull<Header>) -> Task<S> { @@ -434,7 +486,6 @@ unsafe impl<S> linked_list::Link for Task<S> { } unsafe fn pointers(target: NonNull<Header>) -> NonNull<linked_list::Pointers<Header>> { - // Not super great as it avoids some of looms checking... - NonNull::from(target.as_ref().owned.with_mut(|ptr| &mut *ptr)) + self::core::Trailer::addr_of_owned(Header::get_trailer(target)) } } diff --git a/src/runtime/task/raw.rs b/src/runtime/task/raw.rs index fbc9574..b9700ae 100644 --- a/src/runtime/task/raw.rs +++ b/src/runtime/task/raw.rs @@ -1,5 +1,6 @@ use crate::future::Future; -use crate::runtime::task::{Cell, Harness, Header, Schedule, State}; +use crate::runtime::task::core::{Core, Trailer}; +use crate::runtime::task::{Cell, Harness, Header, Id, Schedule, State}; use std::ptr::NonNull; use std::task::{Poll, Waker}; @@ -13,6 +14,9 @@ pub(super) struct Vtable { /// Polls the future. pub(super) poll: unsafe fn(NonNull<Header>), + /// Schedules the task for execution on the runtime. + pub(super) schedule: unsafe fn(NonNull<Header>), + /// Deallocates the memory. pub(super) dealloc: unsafe fn(NonNull<Header>), @@ -22,32 +26,142 @@ pub(super) struct Vtable { /// The join handle has been dropped. pub(super) drop_join_handle_slow: unsafe fn(NonNull<Header>), - /// The task is remotely aborted. - pub(super) remote_abort: unsafe fn(NonNull<Header>), + /// An abort handle has been dropped. + pub(super) drop_abort_handle: unsafe fn(NonNull<Header>), /// Scheduler is being shutdown. pub(super) shutdown: unsafe fn(NonNull<Header>), + + /// The number of bytes that the `trailer` field is offset from the header. + pub(super) trailer_offset: usize, + + /// The number of bytes that the `scheduler` field is offset from the header. + pub(super) scheduler_offset: usize, + + /// The number of bytes that the `id` field is offset from the header. + pub(super) id_offset: usize, } /// Get the vtable for the requested `T` and `S` generics. pub(super) fn vtable<T: Future, S: Schedule>() -> &'static Vtable { &Vtable { poll: poll::<T, S>, + schedule: schedule::<S>, dealloc: dealloc::<T, S>, try_read_output: try_read_output::<T, S>, drop_join_handle_slow: drop_join_handle_slow::<T, S>, - remote_abort: remote_abort::<T, S>, + drop_abort_handle: drop_abort_handle::<T, S>, shutdown: shutdown::<T, S>, + trailer_offset: OffsetHelper::<T, S>::TRAILER_OFFSET, + scheduler_offset: OffsetHelper::<T, S>::SCHEDULER_OFFSET, + id_offset: OffsetHelper::<T, S>::ID_OFFSET, } } +/// Calling `get_trailer_offset` directly in vtable doesn't work because it +/// prevents the vtable from being promoted to a static reference. +/// +/// See this thread for more info: +/// <https://users.rust-lang.org/t/custom-vtables-with-integers/78508> +struct OffsetHelper<T, S>(T, S); +impl<T: Future, S: Schedule> OffsetHelper<T, S> { + // Pass `size_of`/`align_of` as arguments rather than calling them directly + // inside `get_trailer_offset` because trait bounds on generic parameters + // of const fn are unstable on our MSRV. + const TRAILER_OFFSET: usize = get_trailer_offset( + std::mem::size_of::<Header>(), + std::mem::size_of::<Core<T, S>>(), + std::mem::align_of::<Core<T, S>>(), + std::mem::align_of::<Trailer>(), + ); + + // The `scheduler` is the first field of `Core`, so it has the same + // offset as `Core`. + const SCHEDULER_OFFSET: usize = get_core_offset( + std::mem::size_of::<Header>(), + std::mem::align_of::<Core<T, S>>(), + ); + + const ID_OFFSET: usize = get_id_offset( + std::mem::size_of::<Header>(), + std::mem::align_of::<Core<T, S>>(), + std::mem::size_of::<S>(), + std::mem::align_of::<Id>(), + ); +} + +/// Compute the offset of the `Trailer` field in `Cell<T, S>` using the +/// `#[repr(C)]` algorithm. +/// +/// Pseudo-code for the `#[repr(C)]` algorithm can be found here: +/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs> +const fn get_trailer_offset( + header_size: usize, + core_size: usize, + core_align: usize, + trailer_align: usize, +) -> usize { + let mut offset = header_size; + + let core_misalign = offset % core_align; + if core_misalign > 0 { + offset += core_align - core_misalign; + } + offset += core_size; + + let trailer_misalign = offset % trailer_align; + if trailer_misalign > 0 { + offset += trailer_align - trailer_misalign; + } + + offset +} + +/// Compute the offset of the `Core<T, S>` field in `Cell<T, S>` using the +/// `#[repr(C)]` algorithm. +/// +/// Pseudo-code for the `#[repr(C)]` algorithm can be found here: +/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs> +const fn get_core_offset(header_size: usize, core_align: usize) -> usize { + let mut offset = header_size; + + let core_misalign = offset % core_align; + if core_misalign > 0 { + offset += core_align - core_misalign; + } + + offset +} + +/// Compute the offset of the `Id` field in `Cell<T, S>` using the +/// `#[repr(C)]` algorithm. +/// +/// Pseudo-code for the `#[repr(C)]` algorithm can be found here: +/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs> +const fn get_id_offset( + header_size: usize, + core_align: usize, + scheduler_size: usize, + id_align: usize, +) -> usize { + let mut offset = get_core_offset(header_size, core_align); + offset += scheduler_size; + + let id_misalign = offset % id_align; + if id_misalign > 0 { + offset += id_align - id_misalign; + } + + offset +} + impl RawTask { - pub(super) fn new<T, S>(task: T, scheduler: S) -> RawTask + pub(super) fn new<T, S>(task: T, scheduler: S, id: Id) -> RawTask where T: Future, S: Schedule, { - let ptr = Box::into_raw(Cell::<_, S>::new(task, scheduler, State::new())); + let ptr = Box::into_raw(Cell::<_, S>::new(task, scheduler, State::new(), id)); let ptr = unsafe { NonNull::new_unchecked(ptr as *mut Header) }; RawTask { ptr } @@ -57,19 +171,40 @@ impl RawTask { RawTask { ptr } } - /// Returns a reference to the task's meta structure. - /// - /// Safe as `Header` is `Sync`. + pub(super) fn header_ptr(&self) -> NonNull<Header> { + self.ptr + } + + pub(super) fn trailer_ptr(&self) -> NonNull<Trailer> { + unsafe { Header::get_trailer(self.ptr) } + } + + /// Returns a reference to the task's header. pub(super) fn header(&self) -> &Header { unsafe { self.ptr.as_ref() } } + /// Returns a reference to the task's trailer. + pub(super) fn trailer(&self) -> &Trailer { + unsafe { &*self.trailer_ptr().as_ptr() } + } + + /// Returns a reference to the task's state. + pub(super) fn state(&self) -> &State { + &self.header().state + } + /// Safety: mutual exclusion is required to call this function. pub(super) fn poll(self) { let vtable = self.header().vtable; unsafe { (vtable.poll)(self.ptr) } } + pub(super) fn schedule(self) { + let vtable = self.header().vtable; + unsafe { (vtable.schedule)(self.ptr) } + } + pub(super) fn dealloc(self) { let vtable = self.header().vtable; unsafe { @@ -89,14 +224,21 @@ impl RawTask { unsafe { (vtable.drop_join_handle_slow)(self.ptr) } } + pub(super) fn drop_abort_handle(self) { + let vtable = self.header().vtable; + unsafe { (vtable.drop_abort_handle)(self.ptr) } + } + pub(super) fn shutdown(self) { let vtable = self.header().vtable; unsafe { (vtable.shutdown)(self.ptr) } } - pub(super) fn remote_abort(self) { - let vtable = self.header().vtable; - unsafe { (vtable.remote_abort)(self.ptr) } + /// Increment the task's reference count. + /// + /// Currently, this is used only when creating an `AbortHandle`. + pub(super) fn ref_inc(self) { + self.header().state.ref_inc(); } } @@ -113,6 +255,15 @@ unsafe fn poll<T: Future, S: Schedule>(ptr: NonNull<Header>) { harness.poll(); } +unsafe fn schedule<S: Schedule>(ptr: NonNull<Header>) { + use crate::runtime::task::{Notified, Task}; + + let scheduler = Header::get_scheduler::<S>(ptr); + scheduler + .as_ref() + .schedule(Notified(Task::from_raw(ptr.cast()))); +} + unsafe fn dealloc<T: Future, S: Schedule>(ptr: NonNull<Header>) { let harness = Harness::<T, S>::from_raw(ptr); harness.dealloc(); @@ -134,9 +285,9 @@ unsafe fn drop_join_handle_slow<T: Future, S: Schedule>(ptr: NonNull<Header>) { harness.drop_join_handle_slow() } -unsafe fn remote_abort<T: Future, S: Schedule>(ptr: NonNull<Header>) { +unsafe fn drop_abort_handle<T: Future, S: Schedule>(ptr: NonNull<Header>) { let harness = Harness::<T, S>::from_raw(ptr); - harness.remote_abort() + harness.drop_reference(); } unsafe fn shutdown<T: Future, S: Schedule>(ptr: NonNull<Header>) { diff --git a/src/runtime/task/state.rs b/src/runtime/task/state.rs index c2d5b28..7728312 100644 --- a/src/runtime/task/state.rs +++ b/src/runtime/task/state.rs @@ -378,7 +378,7 @@ impl State { pub(super) fn set_join_waker(&self) -> UpdateResult { self.fetch_update(|curr| { assert!(curr.is_join_interested()); - assert!(!curr.has_join_waker()); + assert!(!curr.is_join_waker_set()); if curr.is_complete() { return None; @@ -398,7 +398,7 @@ impl State { pub(super) fn unset_waker(&self) -> UpdateResult { self.fetch_update(|curr| { assert!(curr.is_join_interested()); - assert!(curr.has_join_waker()); + assert!(curr.is_join_waker_set()); if curr.is_complete() { return None; @@ -546,7 +546,7 @@ impl Snapshot { self.0 &= !JOIN_INTEREST } - pub(super) fn has_join_waker(self) -> bool { + pub(super) fn is_join_waker_set(self) -> bool { self.0 & JOIN_WAKER == JOIN_WAKER } @@ -588,7 +588,7 @@ impl fmt::Debug for Snapshot { .field("is_notified", &self.is_notified()) .field("is_cancelled", &self.is_cancelled()) .field("is_join_interested", &self.is_join_interested()) - .field("has_join_waker", &self.has_join_waker()) + .field("is_join_waker_set", &self.is_join_waker_set()) .field("ref_count", &self.ref_count()) .finish() } diff --git a/src/runtime/task/waker.rs b/src/runtime/task/waker.rs index b7313b4..b5f5ace 100644 --- a/src/runtime/task/waker.rs +++ b/src/runtime/task/waker.rs @@ -1,6 +1,5 @@ use crate::future::Future; -use crate::runtime::task::harness::Harness; -use crate::runtime::task::{Header, Schedule}; +use crate::runtime::task::{Header, RawTask, Schedule}; use std::marker::PhantomData; use std::mem::ManuallyDrop; @@ -13,9 +12,9 @@ pub(super) struct WakerRef<'a, S: 'static> { _p: PhantomData<(&'a Header, S)>, } -/// Returns a `WakerRef` which avoids having to pre-emptively increase the +/// Returns a `WakerRef` which avoids having to preemptively increase the /// refcount if there is no need to do so. -pub(super) fn waker_ref<T, S>(header: &Header) -> WakerRef<'_, S> +pub(super) fn waker_ref<T, S>(header: &NonNull<Header>) -> WakerRef<'_, S> where T: Future, S: Schedule, @@ -28,7 +27,7 @@ where // point and not an *owned* waker, we must ensure that `drop` is never // called on this waker instance. This is done by wrapping it with // `ManuallyDrop` and then never calling drop. - let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker::<T, S>(header))) }; + let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker(*header))) }; WakerRef { waker, @@ -46,8 +45,8 @@ impl<S> ops::Deref for WakerRef<'_, S> { cfg_trace! { macro_rules! trace { - ($harness:expr, $op:expr) => { - if let Some(id) = $harness.id() { + ($header:expr, $op:expr) => { + if let Some(id) = Header::get_tracing_id(&$header) { tracing::trace!( target: "tokio::task::waker", op = $op, @@ -60,71 +59,46 @@ cfg_trace! { cfg_not_trace! { macro_rules! trace { - ($harness:expr, $op:expr) => { + ($header:expr, $op:expr) => { // noop - let _ = &$harness; + let _ = &$header; } } } -unsafe fn clone_waker<T, S>(ptr: *const ()) -> RawWaker -where - T: Future, - S: Schedule, -{ - let header = ptr as *const Header; - let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::<T, S>::from_raw(ptr); - trace!(harness, "waker.clone"); - (*header).state.ref_inc(); - raw_waker::<T, S>(header) +unsafe fn clone_waker(ptr: *const ()) -> RawWaker { + let header = NonNull::new_unchecked(ptr as *mut Header); + trace!(header, "waker.clone"); + header.as_ref().state.ref_inc(); + raw_waker(header) } -unsafe fn drop_waker<T, S>(ptr: *const ()) -where - T: Future, - S: Schedule, -{ +unsafe fn drop_waker(ptr: *const ()) { let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::<T, S>::from_raw(ptr); - trace!(harness, "waker.drop"); - harness.drop_reference(); + trace!(ptr, "waker.drop"); + let raw = RawTask::from_raw(ptr); + raw.drop_reference(); } -unsafe fn wake_by_val<T, S>(ptr: *const ()) -where - T: Future, - S: Schedule, -{ +unsafe fn wake_by_val(ptr: *const ()) { let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::<T, S>::from_raw(ptr); - trace!(harness, "waker.wake"); - harness.wake_by_val(); + trace!(ptr, "waker.wake"); + let raw = RawTask::from_raw(ptr); + raw.wake_by_val(); } // Wake without consuming the waker -unsafe fn wake_by_ref<T, S>(ptr: *const ()) -where - T: Future, - S: Schedule, -{ +unsafe fn wake_by_ref(ptr: *const ()) { let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::<T, S>::from_raw(ptr); - trace!(harness, "waker.wake_by_ref"); - harness.wake_by_ref(); + trace!(ptr, "waker.wake_by_ref"); + let raw = RawTask::from_raw(ptr); + raw.wake_by_ref(); } -fn raw_waker<T, S>(header: *const Header) -> RawWaker -where - T: Future, - S: Schedule, -{ - let ptr = header as *const (); - let vtable = &RawWakerVTable::new( - clone_waker::<T, S>, - wake_by_val::<T, S>, - wake_by_ref::<T, S>, - drop_waker::<T, S>, - ); - RawWaker::new(ptr, vtable) +static WAKER_VTABLE: RawWakerVTable = + RawWakerVTable::new(clone_waker, wake_by_val, wake_by_ref, drop_waker); + +fn raw_waker(header: NonNull<Header>) -> RawWaker { + let ptr = header.as_ptr() as *const (); + RawWaker::new(ptr, &WAKER_VTABLE) } diff --git a/src/runtime/tests/loom_blocking.rs b/src/runtime/tests/loom_blocking.rs index 8fb54c5..5c4aeae 100644 --- a/src/runtime/tests/loom_blocking.rs +++ b/src/runtime/tests/loom_blocking.rs @@ -23,6 +23,77 @@ fn blocking_shutdown() { }); } +#[test] +fn spawn_mandatory_blocking_should_always_run() { + use crate::runtime::tests::loom_oneshot; + loom::model(|| { + let rt = runtime::Builder::new_current_thread().build().unwrap(); + + let (tx, rx) = loom_oneshot::channel(); + let _enter = rt.enter(); + runtime::spawn_blocking(|| {}); + runtime::spawn_mandatory_blocking(move || { + let _ = tx.send(()); + }) + .unwrap(); + + drop(rt); + + // This call will deadlock if `spawn_mandatory_blocking` doesn't run. + let () = rx.recv(); + }); +} + +#[test] +fn spawn_mandatory_blocking_should_run_even_when_shutting_down_from_other_thread() { + use crate::runtime::tests::loom_oneshot; + loom::model(|| { + let rt = runtime::Builder::new_current_thread().build().unwrap(); + let handle = rt.handle().clone(); + + // Drop the runtime in a different thread + { + loom::thread::spawn(move || { + drop(rt); + }); + } + + let _enter = handle.enter(); + let (tx, rx) = loom_oneshot::channel(); + let handle = runtime::spawn_mandatory_blocking(move || { + let _ = tx.send(()); + }); + + // handle.is_some() means that `spawn_mandatory_blocking` + // promised us to run the blocking task + if handle.is_some() { + // This call will deadlock if `spawn_mandatory_blocking` doesn't run. + let () = rx.recv(); + } + }); +} + +#[test] +fn spawn_blocking_when_paused() { + use std::time::Duration; + loom::model(|| { + let rt = crate::runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .unwrap(); + let handle = rt.handle(); + let _enter = handle.enter(); + let a = crate::task::spawn_blocking(|| {}); + let b = crate::task::spawn_blocking(|| {}); + rt.block_on(crate::time::timeout(Duration::from_millis(1), async move { + a.await.expect("blocking task should finish"); + b.await.expect("blocking task should finish"); + })) + .expect("timeout should not trigger"); + }); +} + fn mk_runtime(num_threads: usize) -> Runtime { runtime::Builder::new_multi_thread() .worker_threads(num_threads) diff --git a/src/runtime/tests/loom_basic_scheduler.rs b/src/runtime/tests/loom_current_thread_scheduler.rs index d2894b9..a772603 100644 --- a/src/runtime/tests/loom_basic_scheduler.rs +++ b/src/runtime/tests/loom_current_thread_scheduler.rs @@ -34,20 +34,22 @@ fn assert_at_most_num_polls(rt: Arc<Runtime>, at_most_polls: usize) { #[test] fn block_on_num_polls() { loom::model(|| { - // we expect at most 3 number of polls because there are - // three points at which we poll the future. At any of these - // points it can be ready: + // we expect at most 4 number of polls because there are three points at + // which we poll the future and an opportunity for a false-positive.. At + // any of these points it can be ready: // - // - when we fail to steal the parker and we block on a - // notification that it is available. + // - when we fail to steal the parker and we block on a notification + // that it is available. // // - when we steal the parker and we schedule the future // - // - when the future is woken up and we have ran the max - // number of tasks for the current tick or there are no - // more tasks to run. + // - when the future is woken up and we have ran the max number of tasks + // for the current tick or there are no more tasks to run. // - let at_most = 3; + // - a thread is notified that the parker is available but a third + // thread acquires it before the notified thread can. + // + let at_most = 4; let rt1 = Arc::new(Builder::new_current_thread().build().unwrap()); let rt2 = rt1.clone(); diff --git a/src/runtime/tests/loom_join_set.rs b/src/runtime/tests/loom_join_set.rs new file mode 100644 index 0000000..bd34387 --- /dev/null +++ b/src/runtime/tests/loom_join_set.rs @@ -0,0 +1,82 @@ +use crate::runtime::Builder; +use crate::task::JoinSet; + +#[test] +fn test_join_set() { + loom::model(|| { + let rt = Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + let mut set = JoinSet::new(); + + rt.block_on(async { + assert_eq!(set.len(), 0); + set.spawn(async { () }); + assert_eq!(set.len(), 1); + set.spawn(async { () }); + assert_eq!(set.len(), 2); + let () = set.join_next().await.unwrap().unwrap(); + assert_eq!(set.len(), 1); + set.spawn(async { () }); + assert_eq!(set.len(), 2); + let () = set.join_next().await.unwrap().unwrap(); + assert_eq!(set.len(), 1); + let () = set.join_next().await.unwrap().unwrap(); + assert_eq!(set.len(), 0); + set.spawn(async { () }); + assert_eq!(set.len(), 1); + }); + + drop(set); + drop(rt); + }); +} + +#[test] +fn abort_all_during_completion() { + use std::sync::{ + atomic::{AtomicBool, Ordering::SeqCst}, + Arc, + }; + + // These booleans assert that at least one execution had the task complete first, and that at + // least one execution had the task be cancelled before it completed. + let complete_happened = Arc::new(AtomicBool::new(false)); + let cancel_happened = Arc::new(AtomicBool::new(false)); + + { + let complete_happened = complete_happened.clone(); + let cancel_happened = cancel_happened.clone(); + loom::model(move || { + let rt = Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + + let mut set = JoinSet::new(); + + rt.block_on(async { + set.spawn(async { () }); + set.abort_all(); + + match set.join_next().await { + Some(Ok(())) => complete_happened.store(true, SeqCst), + Some(Err(err)) if err.is_cancelled() => cancel_happened.store(true, SeqCst), + Some(Err(err)) => panic!("fail: {}", err), + None => { + unreachable!("Aborting the task does not remove it from the JoinSet.") + } + } + + assert!(matches!(set.join_next().await, None)); + }); + + drop(set); + drop(rt); + }); + } + + assert!(complete_happened.load(SeqCst)); + assert!(cancel_happened.load(SeqCst)); +} diff --git a/src/runtime/tests/loom_queue.rs b/src/runtime/tests/loom_queue.rs index 2cbb0a1..fc93bf3 100644 --- a/src/runtime/tests/loom_queue.rs +++ b/src/runtime/tests/loom_queue.rs @@ -1,7 +1,7 @@ -use crate::runtime::blocking::NoopSchedule; -use crate::runtime::queue; -use crate::runtime::stats::WorkerStatsBatcher; +use crate::runtime::scheduler::multi_thread::queue; use crate::runtime::task::Inject; +use crate::runtime::tests::NoopSchedule; +use crate::runtime::MetricsBatch; use loom::thread; @@ -10,14 +10,15 @@ fn basic() { loom::model(|| { let (steal, mut local) = queue::local(); let inject = Inject::new(); + let mut metrics = MetricsBatch::new(); let th = thread::spawn(move || { - let mut stats = WorkerStatsBatcher::new(0); + let mut metrics = MetricsBatch::new(); let (_, mut local) = queue::local(); let mut n = 0; for _ in 0..3 { - if steal.steal_into(&mut local, &mut stats).is_some() { + if steal.steal_into(&mut local, &mut metrics).is_some() { n += 1; } @@ -34,7 +35,7 @@ fn basic() { for _ in 0..2 { for _ in 0..2 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); } if local.pop().is_some() { @@ -43,7 +44,7 @@ fn basic() { // Push another task let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); while local.pop().is_some() { n += 1; @@ -65,13 +66,14 @@ fn steal_overflow() { loom::model(|| { let (steal, mut local) = queue::local(); let inject = Inject::new(); + let mut metrics = MetricsBatch::new(); let th = thread::spawn(move || { - let mut stats = WorkerStatsBatcher::new(0); + let mut metrics = MetricsBatch::new(); let (_, mut local) = queue::local(); let mut n = 0; - if steal.steal_into(&mut local, &mut stats).is_some() { + if steal.steal_into(&mut local, &mut metrics).is_some() { n += 1; } @@ -86,7 +88,7 @@ fn steal_overflow() { // push a task, pop a task let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); if local.pop().is_some() { n += 1; @@ -94,7 +96,7 @@ fn steal_overflow() { for _ in 0..6 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); } n += th.join().unwrap(); @@ -116,10 +118,10 @@ fn multi_stealer() { const NUM_TASKS: usize = 5; fn steal_tasks(steal: queue::Steal<NoopSchedule>) -> usize { - let mut stats = WorkerStatsBatcher::new(0); + let mut metrics = MetricsBatch::new(); let (_, mut local) = queue::local(); - if steal.steal_into(&mut local, &mut stats).is_none() { + if steal.steal_into(&mut local, &mut metrics).is_none() { return 0; } @@ -135,11 +137,12 @@ fn multi_stealer() { loom::model(|| { let (steal, mut local) = queue::local(); let inject = Inject::new(); + let mut metrics = MetricsBatch::new(); // Push work for _ in 0..NUM_TASKS { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); } let th1 = { @@ -169,7 +172,7 @@ fn multi_stealer() { #[test] fn chained_steal() { loom::model(|| { - let mut stats = WorkerStatsBatcher::new(0); + let mut metrics = MetricsBatch::new(); let (s1, mut l1) = queue::local(); let (s2, mut l2) = queue::local(); let inject = Inject::new(); @@ -177,17 +180,17 @@ fn chained_steal() { // Load up some tasks for _ in 0..4 { let (task, _) = super::unowned(async {}); - l1.push_back(task, &inject); + l1.push_back(task, &inject, &mut metrics); let (task, _) = super::unowned(async {}); - l2.push_back(task, &inject); + l2.push_back(task, &inject, &mut metrics); } // Spawn a task to steal from **our** queue let th = thread::spawn(move || { - let mut stats = WorkerStatsBatcher::new(0); + let mut metrics = MetricsBatch::new(); let (_, mut local) = queue::local(); - s1.steal_into(&mut local, &mut stats); + s1.steal_into(&mut local, &mut metrics); while local.pop().is_some() {} }); @@ -195,7 +198,7 @@ fn chained_steal() { // Drain our tasks, then attempt to steal while l1.pop().is_some() {} - s2.steal_into(&mut l1, &mut stats); + s2.steal_into(&mut l1, &mut metrics); th.join().unwrap(); diff --git a/src/runtime/tests/loom_yield.rs b/src/runtime/tests/loom_yield.rs new file mode 100644 index 0000000..ba506e5 --- /dev/null +++ b/src/runtime/tests/loom_yield.rs @@ -0,0 +1,37 @@ +use crate::runtime::park; +use crate::runtime::tests::loom_oneshot as oneshot; +use crate::runtime::{self, Runtime}; + +#[test] +fn yield_calls_park_before_scheduling_again() { + // Don't need to check all permutations + let mut loom = loom::model::Builder::default(); + loom.max_permutations = Some(1); + loom.check(|| { + let rt = mk_runtime(2); + let (tx, rx) = oneshot::channel::<()>(); + + rt.spawn(async { + let tid = loom::thread::current().id(); + let park_count = park::current_thread_park_count(); + + crate::task::yield_now().await; + + if tid == loom::thread::current().id() { + let new_park_count = park::current_thread_park_count(); + assert_eq!(park_count + 1, new_park_count); + } + + tx.send(()); + }); + + rx.recv(); + }); +} + +fn mk_runtime(num_threads: usize) -> Runtime { + runtime::Builder::new_multi_thread() + .worker_threads(num_threads) + .build() + .unwrap() +} diff --git a/src/runtime/tests/mod.rs b/src/runtime/tests/mod.rs index be36d6f..4e7c245 100644 --- a/src/runtime/tests/mod.rs +++ b/src/runtime/tests/mod.rs @@ -1,8 +1,30 @@ +// Enable dead_code / unreachable_pub here. It has been disabled in lib.rs for +// other code when running loom tests. +#![cfg_attr(loom, warn(dead_code, unreachable_pub))] + +use self::noop_scheduler::NoopSchedule; use self::unowned_wrapper::unowned; +mod noop_scheduler { + use crate::runtime::task::{self, Task}; + + /// `task::Schedule` implementation that does nothing, for testing. + pub(crate) struct NoopSchedule; + + impl task::Schedule for NoopSchedule { + fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> { + None + } + + fn schedule(&self, _task: task::Notified<Self>) { + unreachable!(); + } + } +} + mod unowned_wrapper { - use crate::runtime::blocking::NoopSchedule; - use crate::runtime::task::{JoinHandle, Notified}; + use crate::runtime::task::{Id, JoinHandle, Notified}; + use crate::runtime::tests::NoopSchedule; #[cfg(all(tokio_unstable, feature = "tracing"))] pub(crate) fn unowned<T>(task: T) -> (Notified<NoopSchedule>, JoinHandle<T::Output>) @@ -13,7 +35,7 @@ mod unowned_wrapper { use tracing::Instrument; let span = tracing::trace_span!("test_span"); let task = task.instrument(span); - let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule); + let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule, Id::next()); (task.into_notified(), handle) } @@ -23,19 +45,21 @@ mod unowned_wrapper { T: std::future::Future + Send + 'static, T::Output: Send + 'static, { - let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule); + let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule, Id::next()); (task.into_notified(), handle) } } cfg_loom! { - mod loom_basic_scheduler; - mod loom_local; mod loom_blocking; + mod loom_current_thread_scheduler; + mod loom_local; mod loom_oneshot; mod loom_pool; mod loom_queue; mod loom_shutdown_join; + mod loom_join_set; + mod loom_yield; } cfg_not_loom! { diff --git a/src/runtime/tests/queue.rs b/src/runtime/tests/queue.rs index 47f1b01..68d2e89 100644 --- a/src/runtime/tests/queue.rs +++ b/src/runtime/tests/queue.rs @@ -1,18 +1,39 @@ -use crate::runtime::queue; -use crate::runtime::stats::WorkerStatsBatcher; +use crate::runtime::scheduler::multi_thread::queue; use crate::runtime::task::{self, Inject, Schedule, Task}; +use crate::runtime::MetricsBatch; use std::thread; use std::time::Duration; +#[allow(unused)] +macro_rules! assert_metrics { + ($metrics:ident, $field:ident == $v:expr) => {{ + use crate::runtime::WorkerMetrics; + use std::sync::atomic::Ordering::Relaxed; + + let worker = WorkerMetrics::new(); + $metrics.submit(&worker); + + let expect = $v; + let actual = worker.$field.load(Relaxed); + + assert!(actual == expect, "expect = {}; actual = {}", expect, actual) + }}; +} + #[test] fn fits_256() { let (_, mut local) = queue::local(); let inject = Inject::new(); + let mut metrics = MetricsBatch::new(); for _ in 0..256 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); + } + + cfg_metrics! { + assert_metrics!(metrics, overflow_count == 0); } assert!(inject.pop().is_none()); @@ -24,10 +45,15 @@ fn fits_256() { fn overflow() { let (_, mut local) = queue::local(); let inject = Inject::new(); + let mut metrics = MetricsBatch::new(); for _ in 0..257 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); + } + + cfg_metrics! { + assert_metrics!(metrics, overflow_count == 1); } let mut n = 0; @@ -45,7 +71,7 @@ fn overflow() { #[test] fn steal_batch() { - let mut stats = WorkerStatsBatcher::new(0); + let mut metrics = MetricsBatch::new(); let (steal1, mut local1) = queue::local(); let (_, mut local2) = queue::local(); @@ -53,10 +79,14 @@ fn steal_batch() { for _ in 0..4 { let (task, _) = super::unowned(async {}); - local1.push_back(task, &inject); + local1.push_back(task, &inject, &mut metrics); } - assert!(steal1.steal_into(&mut local2, &mut stats).is_some()); + assert!(steal1.steal_into(&mut local2, &mut metrics).is_some()); + + cfg_metrics! { + assert_metrics!(metrics, steal_count == 2); + } for _ in 0..1 { assert!(local2.pop().is_some()); @@ -71,25 +101,35 @@ fn steal_batch() { assert!(local1.pop().is_none()); } +const fn normal_or_miri(normal: usize, miri: usize) -> usize { + if cfg!(miri) { + miri + } else { + normal + } +} + #[test] fn stress1() { - const NUM_ITER: usize = 1; - const NUM_STEAL: usize = 1_000; - const NUM_LOCAL: usize = 1_000; - const NUM_PUSH: usize = 500; - const NUM_POP: usize = 250; + const NUM_ITER: usize = 5; + const NUM_STEAL: usize = normal_or_miri(1_000, 10); + const NUM_LOCAL: usize = normal_or_miri(1_000, 10); + const NUM_PUSH: usize = normal_or_miri(500, 10); + const NUM_POP: usize = normal_or_miri(250, 10); + + let mut metrics = MetricsBatch::new(); for _ in 0..NUM_ITER { let (steal, mut local) = queue::local(); let inject = Inject::new(); let th = thread::spawn(move || { - let mut stats = WorkerStatsBatcher::new(0); + let mut metrics = MetricsBatch::new(); let (_, mut local) = queue::local(); let mut n = 0; for _ in 0..NUM_STEAL { - if steal.steal_into(&mut local, &mut stats).is_some() { + if steal.steal_into(&mut local, &mut metrics).is_some() { n += 1; } @@ -100,6 +140,10 @@ fn stress1() { thread::yield_now(); } + cfg_metrics! { + assert_metrics!(metrics, steal_count == n as _); + } + n }); @@ -108,7 +152,7 @@ fn stress1() { for _ in 0..NUM_LOCAL { for _ in 0..NUM_PUSH { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); } for _ in 0..NUM_POP { @@ -133,15 +177,17 @@ fn stress1() { #[test] fn stress2() { const NUM_ITER: usize = 1; - const NUM_TASKS: usize = 1_000_000; - const NUM_STEAL: usize = 1_000; + const NUM_TASKS: usize = normal_or_miri(1_000_000, 50); + const NUM_STEAL: usize = normal_or_miri(1_000, 10); + + let mut metrics = MetricsBatch::new(); for _ in 0..NUM_ITER { let (steal, mut local) = queue::local(); let inject = Inject::new(); let th = thread::spawn(move || { - let mut stats = WorkerStatsBatcher::new(0); + let mut stats = MetricsBatch::new(); let (_, mut local) = queue::local(); let mut n = 0; @@ -164,7 +210,7 @@ fn stress2() { for i in 0..NUM_TASKS { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject); + local.push_back(task, &inject, &mut metrics); if i % 128 == 0 && local.pop().is_some() { num_pop += 1; diff --git a/src/runtime/tests/task.rs b/src/runtime/tests/task.rs index 04e1b56..a79c0f5 100644 --- a/src/runtime/tests/task.rs +++ b/src/runtime/tests/task.rs @@ -1,5 +1,5 @@ -use crate::runtime::blocking::NoopSchedule; -use crate::runtime::task::{self, unowned, JoinHandle, OwnedTasks, Schedule, Task}; +use crate::runtime::task::{self, unowned, Id, JoinHandle, OwnedTasks, Schedule, Task}; +use crate::runtime::tests::NoopSchedule; use crate::util::TryLock; use std::collections::VecDeque; @@ -55,6 +55,7 @@ fn create_drop1() { unreachable!() }, NoopSchedule, + Id::next(), ); drop(notified); handle.assert_not_dropped(); @@ -71,6 +72,7 @@ fn create_drop2() { unreachable!() }, NoopSchedule, + Id::next(), ); drop(join); handle.assert_not_dropped(); @@ -78,6 +80,46 @@ fn create_drop2() { handle.assert_dropped(); } +#[test] +fn drop_abort_handle1() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + Id::next(), + ); + let abort = join.abort_handle(); + drop(join); + handle.assert_not_dropped(); + drop(notified); + handle.assert_not_dropped(); + drop(abort); + handle.assert_dropped(); +} + +#[test] +fn drop_abort_handle2() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + Id::next(), + ); + let abort = join.abort_handle(); + drop(notified); + handle.assert_not_dropped(); + drop(abort); + handle.assert_not_dropped(); + drop(join); + handle.assert_dropped(); +} + // Shutting down through Notified works #[test] fn create_shutdown1() { @@ -88,6 +130,7 @@ fn create_shutdown1() { unreachable!() }, NoopSchedule, + Id::next(), ); drop(join); handle.assert_not_dropped(); @@ -104,6 +147,7 @@ fn create_shutdown2() { unreachable!() }, NoopSchedule, + Id::next(), ); handle.assert_not_dropped(); notified.shutdown(); @@ -113,7 +157,7 @@ fn create_shutdown2() { #[test] fn unowned_poll() { - let (task, _) = unowned(async {}, NoopSchedule); + let (task, _) = unowned(async {}, NoopSchedule, Id::next()); task.run(); } @@ -228,7 +272,7 @@ impl Runtime { T: 'static + Send + Future, T::Output: 'static + Send, { - let (handle, notified) = self.0.owned.bind(future, self.clone()); + let (handle, notified) = self.0.owned.bind(future, self.clone(), Id::next()); if let Some(notified) = notified { self.schedule(notified); diff --git a/src/runtime/tests/task_combinations.rs b/src/runtime/tests/task_combinations.rs index 76ce233..73a20d9 100644 --- a/src/runtime/tests/task_combinations.rs +++ b/src/runtime/tests/task_combinations.rs @@ -1,8 +1,10 @@ +use std::fmt; use std::future::Future; use std::panic; use std::pin::Pin; use std::task::{Context, Poll}; +use crate::runtime::task::AbortHandle; use crate::runtime::Builder; use crate::sync::oneshot; use crate::task::JoinHandle; @@ -56,6 +58,12 @@ enum CombiAbort { AbortedAfterConsumeOutput = 4, } +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiAbortSource { + JoinHandle, + AbortHandle, +} + #[test] fn test_combinations() { let mut rt = &[ @@ -90,6 +98,13 @@ fn test_combinations() { CombiAbort::AbortedAfterFinish, CombiAbort::AbortedAfterConsumeOutput, ]; + let ah = [ + None, + Some(CombiJoinHandle::DropImmediately), + Some(CombiJoinHandle::DropFirstPoll), + Some(CombiJoinHandle::DropAfterNoConsume), + Some(CombiJoinHandle::DropAfterConsume), + ]; for rt in rt.iter().copied() { for ls in ls.iter().copied() { @@ -98,7 +113,34 @@ fn test_combinations() { for ji in ji.iter().copied() { for jh in jh.iter().copied() { for abort in abort.iter().copied() { - test_combination(rt, ls, task, output, ji, jh, abort); + // abort via join handle --- abort handles + // may be dropped at any point + for ah in ah.iter().copied() { + test_combination( + rt, + ls, + task, + output, + ji, + jh, + ah, + abort, + CombiAbortSource::JoinHandle, + ); + } + // if aborting via AbortHandle, it will + // never be dropped. + test_combination( + rt, + ls, + task, + output, + ji, + jh, + None, + abort, + CombiAbortSource::AbortHandle, + ); } } } @@ -108,6 +150,9 @@ fn test_combinations() { } } +fn is_debug<T: fmt::Debug>(_: &T) {} + +#[allow(clippy::too_many_arguments)] fn test_combination( rt: CombiRuntime, ls: CombiLocalSet, @@ -115,12 +160,24 @@ fn test_combination( output: CombiOutput, ji: CombiJoinInterest, jh: CombiJoinHandle, + ah: Option<CombiJoinHandle>, abort: CombiAbort, + abort_src: CombiAbortSource, ) { - if (jh as usize) < (abort as usize) { - // drop before abort not possible - return; + match (abort_src, ah) { + (CombiAbortSource::JoinHandle, _) if (jh as usize) < (abort as usize) => { + // join handle dropped prior to abort + return; + } + (CombiAbortSource::AbortHandle, Some(_)) => { + // abort handle dropped, we can't abort through the + // abort handle + return; + } + + _ => {} } + if (task == CombiTask::PanicOnDrop) && (output == CombiOutput::PanicOnDrop) { // this causes double panic return; @@ -130,7 +187,15 @@ fn test_combination( return; } - println!("Runtime {:?}, LocalSet {:?}, Task {:?}, Output {:?}, JoinInterest {:?}, JoinHandle {:?}, Abort {:?}", rt, ls, task, output, ji, jh, abort); + is_debug(&rt); + is_debug(&ls); + is_debug(&task); + is_debug(&output); + is_debug(&ji); + is_debug(&jh); + is_debug(&ah); + is_debug(&abort); + is_debug(&abort_src); // A runtime optionally with a LocalSet struct Rt { @@ -282,8 +347,24 @@ fn test_combination( ); } + // If we are either aborting the task via an abort handle, or dropping via + // an abort handle, do that now. + let mut abort_handle = if ah.is_some() || abort_src == CombiAbortSource::AbortHandle { + handle.as_ref().map(JoinHandle::abort_handle) + } else { + None + }; + + let do_abort = |abort_handle: &mut Option<AbortHandle>, + join_handle: Option<&mut JoinHandle<_>>| { + match abort_src { + CombiAbortSource::AbortHandle => abort_handle.take().unwrap().abort(), + CombiAbortSource::JoinHandle => join_handle.unwrap().abort(), + } + }; + if abort == CombiAbort::AbortedImmediately { - handle.as_mut().unwrap().abort(); + do_abort(&mut abort_handle, handle.as_mut()); aborted = true; } if jh == CombiJoinHandle::DropImmediately { @@ -301,12 +382,15 @@ fn test_combination( } if abort == CombiAbort::AbortedFirstPoll { - handle.as_mut().unwrap().abort(); + do_abort(&mut abort_handle, handle.as_mut()); aborted = true; } if jh == CombiJoinHandle::DropFirstPoll { drop(handle.take().unwrap()); } + if ah == Some(CombiJoinHandle::DropFirstPoll) { + drop(abort_handle.take().unwrap()); + } // Signal the future that it can return now let _ = on_complete.send(()); @@ -318,23 +402,42 @@ fn test_combination( if abort == CombiAbort::AbortedAfterFinish { // Don't set aborted to true here as the task already finished - handle.as_mut().unwrap().abort(); + do_abort(&mut abort_handle, handle.as_mut()); } if jh == CombiJoinHandle::DropAfterNoConsume { - // The runtime will usually have dropped every ref-count at this point, - // in which case dropping the JoinHandle drops the output. - // - // (But it might race and still hold a ref-count) - let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + if ah == Some(CombiJoinHandle::DropAfterNoConsume) { drop(handle.take().unwrap()); - })); - if panic.is_err() { - assert!( - (output == CombiOutput::PanicOnDrop) - && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) - && !aborted, - "Dropping JoinHandle shouldn't panic here" - ); + // The runtime will usually have dropped every ref-count at this point, + // in which case dropping the AbortHandle drops the output. + // + // (But it might race and still hold a ref-count) + let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + drop(abort_handle.take().unwrap()); + })); + if panic.is_err() { + assert!( + (output == CombiOutput::PanicOnDrop) + && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) + && !aborted, + "Dropping AbortHandle shouldn't panic here" + ); + } + } else { + // The runtime will usually have dropped every ref-count at this point, + // in which case dropping the JoinHandle drops the output. + // + // (But it might race and still hold a ref-count) + let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + drop(handle.take().unwrap()); + })); + if panic.is_err() { + assert!( + (output == CombiOutput::PanicOnDrop) + && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) + && !aborted, + "Dropping JoinHandle shouldn't panic here" + ); + } } } @@ -362,11 +465,15 @@ fn test_combination( _ => unreachable!(), } - let handle = handle.take().unwrap(); + let mut handle = handle.take().unwrap(); if abort == CombiAbort::AbortedAfterConsumeOutput { - handle.abort(); + do_abort(&mut abort_handle, Some(&mut handle)); } drop(handle); + + if ah == Some(CombiJoinHandle::DropAfterConsume) { + drop(abort_handle.take()); + } } // The output should have been dropped now. Check whether the output diff --git a/src/runtime/thread_id.rs b/src/runtime/thread_id.rs new file mode 100644 index 0000000..ef39289 --- /dev/null +++ b/src/runtime/thread_id.rs @@ -0,0 +1,31 @@ +use std::num::NonZeroU64; + +#[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)] +pub(crate) struct ThreadId(NonZeroU64); + +impl ThreadId { + pub(crate) fn next() -> Self { + use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64}; + + static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(0); + + let mut last = NEXT_ID.load(Relaxed); + loop { + let id = match last.checked_add(1) { + Some(id) => id, + None => exhausted(), + }; + + match NEXT_ID.compare_exchange_weak(last, id, Relaxed, Relaxed) { + Ok(_) => return ThreadId(NonZeroU64::new(id).unwrap()), + Err(id) => last = id, + } + } + } +} + +#[cold] +#[allow(dead_code)] +fn exhausted() -> ! { + panic!("failed to generate unique thread ID: bitspace exhausted") +} diff --git a/src/runtime/thread_pool/mod.rs b/src/runtime/thread_pool/mod.rs deleted file mode 100644 index 82e34c7..0000000 --- a/src/runtime/thread_pool/mod.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! Threadpool - -mod atomic_cell; -use atomic_cell::AtomicCell; - -mod idle; -use self::idle::Idle; - -mod worker; -pub(crate) use worker::Launch; - -pub(crate) use worker::block_in_place; - -use crate::loom::sync::Arc; -use crate::runtime::stats::RuntimeStats; -use crate::runtime::task::JoinHandle; -use crate::runtime::{Callback, Parker}; - -use std::fmt; -use std::future::Future; - -/// Work-stealing based thread pool for executing futures. -pub(crate) struct ThreadPool { - spawner: Spawner, -} - -/// Submits futures to the associated thread pool for execution. -/// -/// A `Spawner` instance is a handle to a single thread pool that allows the owner -/// of the handle to spawn futures onto the thread pool. -/// -/// The `Spawner` handle is *only* used for spawning new futures. It does not -/// impact the lifecycle of the thread pool in any way. The thread pool may -/// shut down while there are outstanding `Spawner` instances. -/// -/// `Spawner` instances are obtained by calling [`ThreadPool::spawner`]. -/// -/// [`ThreadPool::spawner`]: method@ThreadPool::spawner -#[derive(Clone)] -pub(crate) struct Spawner { - shared: Arc<worker::Shared>, -} - -// ===== impl ThreadPool ===== - -impl ThreadPool { - pub(crate) fn new( - size: usize, - parker: Parker, - before_park: Option<Callback>, - after_unpark: Option<Callback>, - ) -> (ThreadPool, Launch) { - let (shared, launch) = worker::create(size, parker, before_park, after_unpark); - let spawner = Spawner { shared }; - let thread_pool = ThreadPool { spawner }; - - (thread_pool, launch) - } - - /// Returns reference to `Spawner`. - /// - /// The `Spawner` handle can be cloned and enables spawning tasks from other - /// threads. - pub(crate) fn spawner(&self) -> &Spawner { - &self.spawner - } - - /// Blocks the current thread waiting for the future to complete. - /// - /// The future will execute on the current thread, but all spawned tasks - /// will be executed on the thread pool. - pub(crate) fn block_on<F>(&self, future: F) -> F::Output - where - F: Future, - { - let mut enter = crate::runtime::enter(true); - enter.block_on(future).expect("failed to park thread") - } -} - -impl fmt::Debug for ThreadPool { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("ThreadPool").finish() - } -} - -impl Drop for ThreadPool { - fn drop(&mut self) { - self.spawner.shutdown(); - } -} - -// ==== impl Spawner ===== - -impl Spawner { - /// Spawns a future onto the thread pool - pub(crate) fn spawn<F>(&self, future: F) -> JoinHandle<F::Output> - where - F: crate::future::Future + Send + 'static, - F::Output: Send + 'static, - { - worker::Shared::bind_new_task(&self.shared, future) - } - - pub(crate) fn shutdown(&mut self) { - self.shared.close(); - } - - pub(crate) fn stats(&self) -> &RuntimeStats { - self.shared.stats() - } -} - -impl fmt::Debug for Spawner { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Spawner").finish() - } -} diff --git a/src/time/driver/entry.rs b/src/runtime/time/entry.rs index 9e9f0dc..f0d613a 100644 --- a/src/time/driver/entry.rs +++ b/src/runtime/time/entry.rs @@ -5,9 +5,9 @@ //! //! # Ground rules //! -//! The heart of the timer implementation here is the `TimerShared` structure, -//! shared between the `TimerEntry` and the driver. Generally, we permit access -//! to `TimerShared` ONLY via either 1) a mutable reference to `TimerEntry` or +//! The heart of the timer implementation here is the [`TimerShared`] structure, +//! shared between the [`TimerEntry`] and the driver. Generally, we permit access +//! to [`TimerShared`] ONLY via either 1) a mutable reference to [`TimerEntry`] or //! 2) a held driver lock. //! //! It follows from this that any changes made while holding BOTH 1 and 2 will @@ -36,7 +36,7 @@ //! point than it was originally scheduled for. //! //! This is accomplished by lazily rescheduling timers. That is, we update the -//! state field field with the true expiration of the timer from the holder of +//! state field with the true expiration of the timer from the holder of //! the [`TimerEntry`]. When the driver services timers (ie, whenever it's //! walking lists of timers), it checks this "true when" value, and reschedules //! based on it. @@ -49,19 +49,20 @@ //! There is of course a race condition between timer reset and timer //! expiration. If the driver fails to observe the updated expiration time, it //! could trigger expiration of the timer too early. However, because -//! `mark_pending` performs a compare-and-swap, it will identify this race and +//! [`mark_pending`][mark_pending] performs a compare-and-swap, it will identify this race and //! refuse to mark the timer as pending. +//! +//! [mark_pending]: TimerHandle::mark_pending use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::AtomicU64; use crate::loom::sync::atomic::Ordering; +use crate::runtime::scheduler; use crate::sync::AtomicWaker; use crate::time::Instant; use crate::util::linked_list; -use super::Handle; - use std::cell::UnsafeCell as StdUnsafeCell; use std::task::{Context, Poll, Waker}; use std::{marker::PhantomPinned, pin::Pin, ptr::NonNull}; @@ -171,7 +172,12 @@ impl StateCell { let mut cur_state = self.state.load(Ordering::Relaxed); loop { - debug_assert!(cur_state < STATE_MIN_VALUE); + // improve the error message for things like + // https://github.com/tokio-rs/tokio/issues/3675 + assert!( + cur_state < STATE_MIN_VALUE, + "mark_pending called when the timer entry is in an invalid state" + ); if cur_state > not_after { break Err(cur_state); @@ -281,10 +287,10 @@ impl StateCell { /// timer. As this participates in intrusive data structures, it must be pinned /// before polling. #[derive(Debug)] -pub(super) struct TimerEntry { - /// Arc reference to the driver. We can only free the driver after +pub(crate) struct TimerEntry { + /// Arc reference to the runtime handle. We can only free the driver after /// deregistering everything from their respective timer wheels. - driver: Handle, + driver: scheduler::Handle, /// Shared inner structure; this is part of an intrusive linked list, and /// therefore other references can exist to it while mutable references to /// Entry exist. @@ -324,18 +330,27 @@ pub(super) type EntryList = crate::util::linked_list::LinkedList<TimerShared, Ti /// /// Note that this structure is located inside the `TimerEntry` structure. #[derive(Debug)] +#[repr(C)] pub(crate) struct TimerShared { + /// Data manipulated by the driver thread itself, only. + driver_state: CachePadded<TimerSharedPadded>, + /// Current state. This records whether the timer entry is currently under /// the ownership of the driver, and if not, its current state (not /// complete, fired, error, etc). state: StateCell, - /// Data manipulated by the driver thread itself, only. - driver_state: CachePadded<TimerSharedPadded>, - _p: PhantomPinned, } +generate_addr_of_methods! { + impl<> TimerShared { + unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<TimerShared>> { + &self.driver_state.0.pointers + } + } +} + impl TimerShared { pub(super) fn new() -> Self { Self { @@ -419,6 +434,12 @@ impl TimerShared { /// frequently to minimize contention. In particular, we move it away from the /// waker, as the waker is updated on every poll. struct TimerSharedPadded { + /// A link within the doubly-linked list of timers on a particular level and + /// slot. Valid only if state is equal to Registered. + /// + /// Only accessed under the entry lock. + pointers: linked_list::Pointers<TimerShared>, + /// The expiration time for which this entry is currently registered. /// Generally owned by the driver, but is accessed by the entry when not /// registered. @@ -426,12 +447,6 @@ struct TimerSharedPadded { /// The true expiration time. Set by the timer future, read by the driver. true_when: AtomicU64, - - /// A link within the doubly-linked list of timers on a particular level and - /// slot. Valid only if state is equal to Registered. - /// - /// Only accessed under the entry lock. - pointers: StdUnsafeCell<linked_list::Pointers<TimerShared>>, } impl std::fmt::Debug for TimerSharedPadded { @@ -448,7 +463,7 @@ impl TimerSharedPadded { Self { cached_when: AtomicU64::new(0), true_when: AtomicU64::new(0), - pointers: StdUnsafeCell::new(linked_list::Pointers::new()), + pointers: linked_list::Pointers::new(), } } } @@ -472,14 +487,18 @@ unsafe impl linked_list::Link for TimerShared { unsafe fn pointers( target: NonNull<Self::Target>, ) -> NonNull<linked_list::Pointers<Self::Target>> { - unsafe { NonNull::new(target.as_ref().driver_state.0.pointers.get()).unwrap() } + TimerShared::addr_of_pointers(target) } } // ===== impl Entry ===== impl TimerEntry { - pub(crate) fn new(handle: &Handle, deadline: Instant) -> Self { + #[track_caller] + pub(crate) fn new(handle: &scheduler::Handle, deadline: Instant) -> Self { + // Panic if the time driver is not enabled + let _ = handle.driver().time(); + let driver = handle.clone(); Self { @@ -522,20 +541,21 @@ impl TimerEntry { // driver did so far and happens-before everything the driver does in // the future. While we have the lock held, we also go ahead and // deregister the entry if necessary. - unsafe { self.driver.clear_entry(NonNull::from(self.inner())) }; + unsafe { self.driver().clear_entry(NonNull::from(self.inner())) }; } pub(crate) fn reset(mut self: Pin<&mut Self>, new_time: Instant) { unsafe { self.as_mut().get_unchecked_mut() }.initial_deadline = None; - let tick = self.driver.time_source().deadline_to_tick(new_time); + let tick = self.driver().time_source().deadline_to_tick(new_time); if self.inner().extend_expiration(tick).is_ok() { return; } unsafe { - self.driver.reregister(tick, self.inner().into()); + self.driver() + .reregister(&self.driver.driver().io, tick, self.inner().into()); } } @@ -543,7 +563,7 @@ impl TimerEntry { mut self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll<Result<(), super::Error>> { - if self.driver.is_shutdown() { + if self.driver().is_shutdown() { panic!("{}", crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR); } @@ -555,6 +575,10 @@ impl TimerEntry { this.inner().state.poll(cx.waker()) } + + pub(crate) fn driver(&self) -> &super::Handle { + self.driver.driver().time() + } } impl TimerHandle { @@ -623,7 +647,73 @@ impl Drop for TimerEntry { } } -#[cfg_attr(target_arch = "x86_64", repr(align(128)))] -#[cfg_attr(not(target_arch = "x86_64"), repr(align(64)))] +// Copied from [crossbeam/cache_padded](https://github.com/crossbeam-rs/crossbeam/blob/fa35346b7c789bba045ad789e894c68c466d1779/crossbeam-utils/src/cache_padded.rs#L62-L127) +// +// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache +// lines at a time, so we have to align to 128 bytes rather than 64. +// +// Sources: +// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf +// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107 +// +// ARM's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size. +// +// Sources: +// - https://www.mono-project.com/news/2016/09/12/arm64-icache/ +// +// powerpc64 has 128-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9 +#[cfg_attr( + any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "powerpc64", + ), + repr(align(128)) +)] +// arm, mips, mips64, and riscv64 have 32-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_riscv64.go#L7 +#[cfg_attr( + any( + target_arch = "arm", + target_arch = "mips", + target_arch = "mips64", + target_arch = "riscv64", + ), + repr(align(32)) +)] +// s390x has 256-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7 +#[cfg_attr(target_arch = "s390x", repr(align(256)))] +// x86 and wasm have 64-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7 +// +// All others are assumed to have 64-byte cache line size. +#[cfg_attr( + not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "arm", + target_arch = "mips", + target_arch = "mips64", + target_arch = "riscv64", + target_arch = "s390x", + )), + repr(align(64)) +)] #[derive(Debug, Default)] struct CachePadded<T>(T); diff --git a/src/runtime/time/handle.rs b/src/runtime/time/handle.rs new file mode 100644 index 0000000..fce791d --- /dev/null +++ b/src/runtime/time/handle.rs @@ -0,0 +1,62 @@ +use crate::runtime::time::TimeSource; +use std::fmt; + +/// Handle to time driver instance. +pub(crate) struct Handle { + pub(super) time_source: TimeSource, + pub(super) inner: super::Inner, +} + +impl Handle { + /// Returns the time source associated with this handle. + pub(crate) fn time_source(&self) -> &TimeSource { + &self.time_source + } + + /// Checks whether the driver has been shutdown. + pub(super) fn is_shutdown(&self) -> bool { + self.inner.is_shutdown() + } + + /// Track that the driver is being unparked + pub(crate) fn unpark(&self) { + #[cfg(feature = "test-util")] + self.inner + .did_wake + .store(true, std::sync::atomic::Ordering::SeqCst); + } +} + +cfg_not_rt! { + impl Handle { + /// Tries to get a handle to the current timer. + /// + /// # Panics + /// + /// This function panics if there is no current timer set. + /// + /// It can be triggered when [`Builder::enable_time`] or + /// [`Builder::enable_all`] are not included in the builder. + /// + /// It can also panic whenever a timer is created outside of a + /// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic, + /// since the function is executed outside of the runtime. + /// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic. + /// And this is because wrapping the function on an async makes it lazy, + /// and so gets executed inside the runtime successfully without + /// panicking. + /// + /// [`Builder::enable_time`]: crate::runtime::Builder::enable_time + /// [`Builder::enable_all`]: crate::runtime::Builder::enable_all + #[track_caller] + pub(crate) fn current() -> Self { + panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) + } + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Handle") + } +} diff --git a/src/time/driver/mod.rs b/src/runtime/time/mod.rs index cf2290b..f81cab8 100644 --- a/src/time/driver/mod.rs +++ b/src/runtime/time/mod.rs @@ -7,22 +7,23 @@ //! Time driver. mod entry; -pub(self) use self::entry::{EntryList, TimerEntry, TimerHandle, TimerShared}; +pub(crate) use entry::TimerEntry; +use entry::{EntryList, TimerHandle, TimerShared}; mod handle; pub(crate) use self::handle::Handle; -mod wheel; +mod source; +pub(crate) use source::TimeSource; -pub(super) mod sleep; +mod wheel; use crate::loom::sync::atomic::{AtomicBool, Ordering}; -use crate::loom::sync::{Arc, Mutex}; -use crate::park::{Park, Unpark}; +use crate::loom::sync::Mutex; +use crate::runtime::driver::{self, IoHandle, IoStack}; use crate::time::error::Error; -use crate::time::{Clock, Duration, Instant}; +use crate::time::{Clock, Duration}; -use std::convert::TryInto; use std::fmt; use std::{num::NonZeroU64, ptr::NonNull, task::Waker}; @@ -82,63 +83,9 @@ use std::{num::NonZeroU64, ptr::NonNull, task::Waker}; /// [timeout]: crate::time::Timeout /// [interval]: crate::time::Interval #[derive(Debug)] -pub(crate) struct Driver<P: Park + 'static> { - /// Timing backend in use. - time_source: ClockTime, - - /// Shared state. - handle: Handle, - +pub(crate) struct Driver { /// Parker to delegate to. - park: P, - - // When `true`, a call to `park_timeout` should immediately return and time - // should not advance. One reason for this to be `true` is if the task - // passed to `Runtime::block_on` called `task::yield_now()`. - // - // While it may look racy, it only has any effect when the clock is paused - // and pausing the clock is restricted to a single-threaded runtime. - #[cfg(feature = "test-util")] - did_wake: Arc<AtomicBool>, -} - -/// A structure which handles conversion from Instants to u64 timestamps. -#[derive(Debug, Clone)] -pub(self) struct ClockTime { - clock: super::clock::Clock, - start_time: Instant, -} - -impl ClockTime { - pub(self) fn new(clock: Clock) -> Self { - Self { - start_time: clock.now(), - clock, - } - } - - pub(self) fn deadline_to_tick(&self, t: Instant) -> u64 { - // Round up to the end of a ms - self.instant_to_tick(t + Duration::from_nanos(999_999)) - } - - pub(self) fn instant_to_tick(&self, t: Instant) -> u64 { - // round up - let dur: Duration = t - .checked_duration_since(self.start_time) - .unwrap_or_else(|| Duration::from_secs(0)); - let ms = dur.as_millis(); - - ms.try_into().expect("Duration too far into the future") - } - - pub(self) fn tick_to_duration(&self, t: u64) -> Duration { - Duration::from_millis(t) - } - - pub(self) fn now(&self) -> u64 { - self.instant_to_tick(self.clock.now()) - } + park: IoStack, } /// Timer state shared between `Driver`, `Handle`, and `Registration`. @@ -148,13 +95,19 @@ struct Inner { /// True if the driver is being shutdown. pub(super) is_shutdown: AtomicBool, + + // When `true`, a call to `park_timeout` should immediately return and time + // should not advance. One reason for this to be `true` is if the task + // passed to `Runtime::block_on` called `task::yield_now()`. + // + // While it may look racy, it only has any effect when the clock is paused + // and pausing the clock is restricted to a single-threaded runtime. + #[cfg(feature = "test-util")] + did_wake: AtomicBool, } /// Time state shared which must be protected by a `Mutex` struct InnerState { - /// Timing backend in use. - time_source: ClockTime, - /// The last published timer `elapsed` value. elapsed: u64, @@ -163,49 +116,67 @@ struct InnerState { /// Timer wheel. wheel: wheel::Wheel, - - /// Unparker that can be used to wake the time driver. - unpark: Box<dyn Unpark>, } // ===== impl Driver ===== -impl<P> Driver<P> -where - P: Park + 'static, -{ +impl Driver { /// Creates a new `Driver` instance that uses `park` to block the current /// thread and `time_source` to get the current time and convert to ticks. /// /// Specifying the source of time is useful when testing. - pub(crate) fn new(park: P, clock: Clock) -> Driver<P> { - let time_source = ClockTime::new(clock); - - let inner = Inner::new(time_source.clone(), Box::new(park.unpark())); + pub(crate) fn new(park: IoStack, clock: Clock) -> (Driver, Handle) { + let time_source = TimeSource::new(clock); - Driver { + let handle = Handle { time_source, - handle: Handle::new(Arc::new(inner)), - park, - #[cfg(feature = "test-util")] - did_wake: Arc::new(AtomicBool::new(false)), - } + inner: Inner { + state: Mutex::new(InnerState { + elapsed: 0, + next_wake: None, + wheel: wheel::Wheel::new(), + }), + is_shutdown: AtomicBool::new(false), + + #[cfg(feature = "test-util")] + did_wake: AtomicBool::new(false), + }, + }; + + let driver = Driver { park }; + + (driver, handle) } - /// Returns a handle to the timer. - /// - /// The `Handle` is how `Sleep` instances are created. The `Sleep` instances - /// can either be created directly or the `Handle` instance can be passed to - /// `with_default`, setting the timer as the default timer for the execution - /// context. - pub(crate) fn handle(&self) -> Handle { - self.handle.clone() + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.park_internal(handle, None) + } + + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + self.park_internal(handle, Some(duration)) } - fn park_internal(&mut self, limit: Option<Duration>) -> Result<(), P::Error> { - let mut lock = self.handle.get().state.lock(); + pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) { + let handle = rt_handle.time(); + + if handle.is_shutdown() { + return; + } - assert!(!self.handle.is_shutdown()); + handle.inner.is_shutdown.store(true, Ordering::SeqCst); + + // Advance time forward to the end of time. + + handle.process_at_time(u64::MAX); + + self.park.shutdown(rt_handle); + } + + fn park_internal(&mut self, rt_handle: &driver::Handle, limit: Option<Duration>) { + let handle = rt_handle.time(); + let mut lock = handle.inner.state.lock(); + + assert!(!handle.is_shutdown()); let next_wake = lock.wheel.next_expiration_time(); lock.next_wake = @@ -215,67 +186,62 @@ where match next_wake { Some(when) => { - let now = self.time_source.now(); + let now = handle.time_source.now(); // Note that we effectively round up to 1ms here - this avoids // very short-duration microsecond-resolution sleeps that the OS // might treat as zero-length. - let mut duration = self.time_source.tick_to_duration(when.saturating_sub(now)); + let mut duration = handle + .time_source + .tick_to_duration(when.saturating_sub(now)); if duration > Duration::from_millis(0) { if let Some(limit) = limit { duration = std::cmp::min(limit, duration); } - self.park_timeout(duration)?; + self.park_thread_timeout(rt_handle, duration); } else { - self.park.park_timeout(Duration::from_secs(0))?; + self.park.park_timeout(rt_handle, Duration::from_secs(0)); } } None => { if let Some(duration) = limit { - self.park_timeout(duration)?; + self.park_thread_timeout(rt_handle, duration); } else { - self.park.park()?; + self.park.park(rt_handle); } } } // Process pending timers after waking up - self.handle.process(); - - Ok(()) + handle.process(); } cfg_test_util! { - fn park_timeout(&mut self, duration: Duration) -> Result<(), P::Error> { - let clock = &self.time_source.clock; + fn park_thread_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) { + let handle = rt_handle.time(); + let clock = &handle.time_source.clock; - if clock.is_paused() { - self.park.park_timeout(Duration::from_secs(0))?; + if clock.can_auto_advance() { + self.park.park_timeout(rt_handle, Duration::from_secs(0)); // If the time driver was woken, then the park completed // before the "duration" elapsed (usually caused by a // yield in `Runtime::block_on`). In this case, we don't // advance the clock. - if !self.did_wake() { + if !handle.did_wake() { // Simulate advancing time clock.advance(duration); } } else { - self.park.park_timeout(duration)?; + self.park.park_timeout(rt_handle, duration); } - - Ok(()) - } - - fn did_wake(&self) -> bool { - self.did_wake.swap(false, Ordering::SeqCst) } } cfg_not_test_util! { - fn park_timeout(&mut self, duration: Duration) -> Result<(), P::Error> { - self.park.park_timeout(duration) + fn park_thread_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) { + self.park.park_timeout(rt_handle, duration); } } } @@ -292,7 +258,7 @@ impl Handle { let mut waker_list: [Option<Waker>; 32] = Default::default(); let mut waker_idx = 0; - let mut lock = self.get().lock(); + let mut lock = self.inner.lock(); if now < lock.elapsed { // Time went backwards! This normally shouldn't happen as the Rust language @@ -323,7 +289,7 @@ impl Handle { waker_idx = 0; - lock = self.get().lock(); + lock = self.inner.lock(); } } } @@ -354,7 +320,7 @@ impl Handle { /// `add_entry` must not be called concurrently. pub(self) unsafe fn clear_entry(&self, entry: NonNull<TimerShared>) { unsafe { - let mut lock = self.get().lock(); + let mut lock = self.inner.lock(); if entry.as_ref().might_be_registered() { lock.wheel.remove(entry); @@ -370,9 +336,14 @@ impl Handle { /// driver. No other threads are allowed to concurrently manipulate the /// timer at all (the current thread should hold an exclusive reference to /// the `TimerEntry`) - pub(self) unsafe fn reregister(&self, new_tick: u64, entry: NonNull<TimerShared>) { + pub(self) unsafe fn reregister( + &self, + unpark: &IoHandle, + new_tick: u64, + entry: NonNull<TimerShared>, + ) { let waker = unsafe { - let mut lock = self.get().lock(); + let mut lock = self.inner.lock(); // We may have raced with a firing/deregistration, so check before // deregistering. @@ -398,12 +369,12 @@ impl Handle { .map(|next_wake| when < next_wake.get()) .unwrap_or(true) { - lock.unpark.unpark(); + unpark.unpark(); } None } - Err((entry, super::error::InsertError::Elapsed)) => unsafe { + Err((entry, crate::time::error::InsertError::Elapsed)) => unsafe { entry.fire(Ok(())) }, } @@ -419,94 +390,17 @@ impl Handle { waker.wake(); } } -} - -impl<P> Park for Driver<P> -where - P: Park + 'static, -{ - type Unpark = TimerUnpark<P>; - type Error = P::Error; - - fn unpark(&self) -> Self::Unpark { - TimerUnpark::new(self) - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.park_internal(None) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.park_internal(Some(duration)) - } - - fn shutdown(&mut self) { - if self.handle.is_shutdown() { - return; - } - - self.handle.get().is_shutdown.store(true, Ordering::SeqCst); - // Advance time forward to the end of time. - - self.handle.process_at_time(u64::MAX); - - self.park.shutdown(); - } -} - -impl<P> Drop for Driver<P> -where - P: Park + 'static, -{ - fn drop(&mut self) { - self.shutdown(); - } -} - -pub(crate) struct TimerUnpark<P: Park + 'static> { - inner: P::Unpark, - - #[cfg(feature = "test-util")] - did_wake: Arc<AtomicBool>, -} - -impl<P: Park + 'static> TimerUnpark<P> { - fn new(driver: &Driver<P>) -> TimerUnpark<P> { - TimerUnpark { - inner: driver.park.unpark(), - - #[cfg(feature = "test-util")] - did_wake: driver.did_wake.clone(), + cfg_test_util! { + fn did_wake(&self) -> bool { + self.inner.did_wake.swap(false, Ordering::SeqCst) } } } -impl<P: Park + 'static> Unpark for TimerUnpark<P> { - fn unpark(&self) { - #[cfg(feature = "test-util")] - self.did_wake.store(true, Ordering::SeqCst); - - self.inner.unpark(); - } -} - // ===== impl Inner ===== impl Inner { - pub(self) fn new(time_source: ClockTime, unpark: Box<dyn Unpark>) -> Self { - Inner { - state: Mutex::new(InnerState { - time_source, - elapsed: 0, - next_wake: None, - unpark, - wheel: wheel::Wheel::new(), - }), - is_shutdown: AtomicBool::new(false), - } - } - /// Locks the driver's inner structure pub(super) fn lock(&self) -> crate::loom::sync::MutexGuard<'_, InnerState> { self.state.lock() diff --git a/src/runtime/time/source.rs b/src/runtime/time/source.rs new file mode 100644 index 0000000..e6788ed --- /dev/null +++ b/src/runtime/time/source.rs @@ -0,0 +1,42 @@ +use crate::time::{Clock, Duration, Instant}; + +use std::convert::TryInto; + +/// A structure which handles conversion from Instants to u64 timestamps. +#[derive(Debug)] +pub(crate) struct TimeSource { + pub(crate) clock: Clock, + start_time: Instant, +} + +impl TimeSource { + pub(crate) fn new(clock: Clock) -> Self { + Self { + start_time: clock.now(), + clock, + } + } + + pub(crate) fn deadline_to_tick(&self, t: Instant) -> u64 { + // Round up to the end of a ms + self.instant_to_tick(t + Duration::from_nanos(999_999)) + } + + pub(crate) fn instant_to_tick(&self, t: Instant) -> u64 { + // round up + let dur: Duration = t + .checked_duration_since(self.start_time) + .unwrap_or_else(|| Duration::from_secs(0)); + let ms = dur.as_millis(); + + ms.try_into().unwrap_or(u64::MAX) + } + + pub(crate) fn tick_to_duration(&self, t: u64) -> Duration { + Duration::from_millis(t) + } + + pub(crate) fn now(&self) -> u64 { + self.instant_to_tick(self.clock.now()) + } +} diff --git a/src/time/driver/tests/mod.rs b/src/runtime/time/tests/mod.rs index 7c5cf1f..88c7d76 100644 --- a/src/time/driver/tests/mod.rs +++ b/src/runtime/time/tests/mod.rs @@ -1,33 +1,27 @@ +#![cfg(not(tokio_wasi))] + use std::{task::Context, time::Duration}; #[cfg(not(loom))] use futures::task::noop_waker_ref; +use crate::loom::sync::atomic::{AtomicBool, Ordering}; use crate::loom::sync::Arc; use crate::loom::thread; -use crate::{ - loom::sync::atomic::{AtomicBool, Ordering}, - park::Unpark, -}; - -use super::{Handle, TimerEntry}; -struct MockUnpark {} -impl Unpark for MockUnpark { - fn unpark(&self) {} -} -impl MockUnpark { - fn mock() -> Box<dyn Unpark> { - Box::new(Self {}) - } -} +use super::TimerEntry; fn block_on<T>(f: impl std::future::Future<Output = T>) -> T { #[cfg(loom)] return loom::future::block_on(f); #[cfg(not(loom))] - return futures::executor::block_on(f); + { + let rt = crate::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(f) + } } fn model(f: impl Fn() + Send + Sync + 'static) { @@ -38,18 +32,26 @@ fn model(f: impl Fn() + Send + Sync + 'static) { f(); } +fn rt(start_paused: bool) -> crate::runtime::Runtime { + crate::runtime::Builder::new_current_thread() + .enable_time() + .start_paused(start_paused) + .build() + .unwrap() +} + #[test] fn single_timer() { model(|| { - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1)); + let entry = TimerEntry::new( + &handle_.inner, + handle_.inner.driver().clock().now() + Duration::from_secs(1), + ); pin!(entry); block_on(futures::future::poll_fn(|cx| { @@ -60,10 +62,12 @@ fn single_timer() { thread::yield_now(); + let handle = handle.inner.driver().time(); + // This may or may not return Some (depending on how it races with the // thread). If it does return None, however, the timer should complete // synchronously. - handle.process_at_time(time_source.now() + 2_000_000_000); + handle.process_at_time(handle.time_source().now() + 2_000_000_000); jh.join().unwrap(); }) @@ -72,15 +76,15 @@ fn single_timer() { #[test] fn drop_timer() { model(|| { - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1)); + let entry = TimerEntry::new( + &handle_.inner, + handle_.inner.driver().clock().now() + Duration::from_secs(1), + ); pin!(entry); let _ = entry @@ -93,8 +97,10 @@ fn drop_timer() { thread::yield_now(); + let handle = handle.inner.driver().time(); + // advance 2s in the future. - handle.process_at_time(time_source.now() + 2_000_000_000); + handle.process_at_time(handle.time_source().now() + 2_000_000_000); jh.join().unwrap(); }) @@ -103,15 +109,15 @@ fn drop_timer() { #[test] fn change_waker() { model(|| { - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1)); + let entry = TimerEntry::new( + &handle_.inner, + handle_.inner.driver().clock().now() + Duration::from_secs(1), + ); pin!(entry); let _ = entry @@ -126,8 +132,10 @@ fn change_waker() { thread::yield_now(); + let handle = handle.inner.driver().time(); + // advance 2s - handle.process_at_time(time_source.now() + 2_000_000_000); + handle.process_at_time(handle.time_source().now() + 2_000_000_000); jh.join().unwrap(); }) @@ -138,18 +146,15 @@ fn reset_future() { model(|| { let finished_early = Arc::new(AtomicBool::new(false)); - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let finished_early_ = finished_early.clone(); - let start = clock.now(); + let start = handle.inner.driver().clock().now(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, start + Duration::from_secs(1)); + let entry = TimerEntry::new(&handle_.inner, start + Duration::from_secs(1)); pin!(entry); let _ = entry @@ -169,12 +174,22 @@ fn reset_future() { thread::yield_now(); + let handle = handle.inner.driver().time(); + // This may or may not return a wakeup time. - handle.process_at_time(time_source.instant_to_tick(start + Duration::from_millis(1500))); + handle.process_at_time( + handle + .time_source() + .instant_to_tick(start + Duration::from_millis(1500)), + ); assert!(!finished_early.load(Ordering::Relaxed)); - handle.process_at_time(time_source.instant_to_tick(start + Duration::from_millis(2500))); + handle.process_at_time( + handle + .time_source() + .instant_to_tick(start + Duration::from_millis(2500)), + ); jh.join().unwrap(); @@ -182,23 +197,27 @@ fn reset_future() { }) } +#[cfg(not(loom))] +fn normal_or_miri<T>(normal: T, miri: T) -> T { + if cfg!(miri) { + miri + } else { + normal + } +} + #[test] #[cfg(not(loom))] fn poll_process_levels() { - let clock = crate::time::clock::Clock::new(true, false); - clock.pause(); - - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source, MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(true); + let handle = rt.handle(); let mut entries = vec![]; - for i in 0..1024 { + for i in 0..normal_or_miri(1024, 64) { let mut entry = Box::pin(TimerEntry::new( - &handle, - clock.now() + Duration::from_millis(i), + &handle.inner, + handle.inner.driver().clock().now() + Duration::from_millis(i), )); let _ = entry @@ -208,8 +227,9 @@ fn poll_process_levels() { entries.push(entry); } - for t in 1..1024 { - handle.process_at_time(t as u64); + for t in 1..normal_or_miri(1024, 64) { + handle.inner.driver().time().process_at_time(t as u64); + for (deadline, future) in entries.iter_mut().enumerate() { let mut context = Context::from_waker(noop_waker_ref()); if deadline <= t { @@ -226,62 +246,19 @@ fn poll_process_levels() { fn poll_process_levels_targeted() { let mut context = Context::from_waker(noop_waker_ref()); - let clock = crate::time::clock::Clock::new(true, false); - clock.pause(); - - let time_source = super::ClockTime::new(clock.clone()); + let rt = rt(true); + let handle = rt.handle(); - let inner = super::Inner::new(time_source, MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); - - let e1 = TimerEntry::new(&handle, clock.now() + Duration::from_millis(193)); + let e1 = TimerEntry::new( + &handle.inner, + handle.inner.driver().clock().now() + Duration::from_millis(193), + ); pin!(e1); + let handle = handle.inner.driver().time(); + handle.process_at_time(62); assert!(e1.as_mut().poll_elapsed(&mut context).is_pending()); handle.process_at_time(192); handle.process_at_time(192); } - -/* -#[test] -fn balanced_incr_and_decr() { - const OPS: usize = 5; - - fn incr(inner: Arc<Inner>) { - for _ in 0..OPS { - inner.increment().expect("increment should not have failed"); - thread::yield_now(); - } - } - - fn decr(inner: Arc<Inner>) { - let mut ops_performed = 0; - while ops_performed < OPS { - if inner.num(Ordering::Relaxed) > 0 { - ops_performed += 1; - inner.decrement(); - } - thread::yield_now(); - } - } - - loom::model(|| { - let unpark = Box::new(MockUnpark); - let instant = Instant::now(); - - let inner = Arc::new(Inner::new(instant, unpark)); - - let incr_inner = inner.clone(); - let decr_inner = inner.clone(); - - let incr_hndle = thread::spawn(move || incr(incr_inner)); - let decr_hndle = thread::spawn(move || decr(decr_inner)); - - incr_hndle.join().expect("should never fail"); - decr_hndle.join().expect("should never fail"); - - assert_eq!(inner.num(Ordering::SeqCst), 0); - }) -} -*/ diff --git a/src/time/driver/wheel/level.rs b/src/runtime/time/wheel/level.rs index 34d3176..7e48ff5 100644 --- a/src/time/driver/wheel/level.rs +++ b/src/runtime/time/wheel/level.rs @@ -1,6 +1,4 @@ -use crate::time::driver::TimerHandle; - -use crate::time::driver::{EntryList, TimerShared}; +use crate::runtime::time::{EntryList, TimerHandle, TimerShared}; use std::{fmt, ptr::NonNull}; @@ -53,7 +51,7 @@ impl Level { // However, that is only supported for arrays of size // 32 or fewer. So in our case we have to explicitly // invoke the constructor for each array element. - let ctor = || EntryList::default(); + let ctor = EntryList::default; Level { level, @@ -143,8 +141,9 @@ impl Level { let level_range = level_range(self.level); let slot_range = slot_range(self.level); - // TODO: This can probably be simplified w/ power of 2 math - let level_start = now - (now % level_range); + // Compute the start date of the current level by masking the low bits + // of `now` (`level_range` is a power of 2). + let level_start = now & !(level_range - 1); let mut deadline = level_start + slot as u64 * slot_range; if deadline <= now { diff --git a/src/time/driver/wheel/mod.rs b/src/runtime/time/wheel/mod.rs index f088f2c..c3ba364 100644 --- a/src/time/driver/wheel/mod.rs +++ b/src/runtime/time/wheel/mod.rs @@ -1,4 +1,4 @@ -use crate::time::driver::{TimerHandle, TimerShared}; +use crate::runtime::time::{TimerHandle, TimerShared}; use crate::time::error::InsertError; mod level; diff --git a/src/signal/mod.rs b/src/signal/mod.rs index 882218a..3aacc60 100644 --- a/src/signal/mod.rs +++ b/src/signal/mod.rs @@ -48,7 +48,7 @@ use std::task::{Context, Poll}; mod ctrl_c; pub use ctrl_c::ctrl_c; -mod registry; +pub(crate) mod registry; mod os { #[cfg(unix)] diff --git a/src/signal/registry.rs b/src/signal/registry.rs index e0a2df9..48e98c8 100644 --- a/src/signal/registry.rs +++ b/src/signal/registry.rs @@ -1,12 +1,10 @@ #![allow(clippy::unit_arg)] use crate::signal::os::{OsExtraData, OsStorage}; - use crate::sync::watch; +use crate::util::once_cell::OnceCell; -use once_cell::sync::Lazy; use std::ops; -use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; pub(crate) type EventId = usize; @@ -152,19 +150,25 @@ impl Globals { } } -pub(crate) fn globals() -> Pin<&'static Globals> +fn globals_init() -> Globals where OsExtraData: 'static + Send + Sync + Init, OsStorage: 'static + Send + Sync + Init, { - static GLOBALS: Lazy<Pin<Box<Globals>>> = Lazy::new(|| { - Box::pin(Globals { - extra: OsExtraData::init(), - registry: Registry::new(OsStorage::init()), - }) - }); - - GLOBALS.as_ref() + Globals { + extra: OsExtraData::init(), + registry: Registry::new(OsStorage::init()), + } +} + +pub(crate) fn globals() -> &'static Globals +where + OsExtraData: 'static + Send + Sync + Init, + OsStorage: 'static + Send + Sync + Init, +{ + static GLOBALS: OnceCell<Globals> = OnceCell::new(); + + GLOBALS.get(globals_init) } #[cfg(all(test, not(loom)))] @@ -202,7 +206,12 @@ mod tests { registry.broadcast(); // Yield so the previous broadcast can get received - crate::time::sleep(std::time::Duration::from_millis(10)).await; + // + // This yields many times since the block_on task is only polled every 61 + // ticks. + for _ in 0..100 { + crate::task::yield_now().await; + } // Send subsequent signal registry.record_event(0); @@ -232,7 +241,7 @@ mod tests { #[test] fn record_invalid_event_does_nothing() { let registry = Registry::new(vec![EventInfo::default()]); - registry.record_event(42); + registry.record_event(1302); } #[test] diff --git a/src/signal/unix.rs b/src/signal/unix.rs index 86ea9a9..e5345fd 100644 --- a/src/signal/unix.rs +++ b/src/signal/unix.rs @@ -6,29 +6,31 @@ #![cfg(unix)] #![cfg_attr(docsrs, doc(cfg(all(unix, feature = "signal"))))] +use crate::runtime::scheduler; +use crate::runtime::signal::Handle; use crate::signal::registry::{globals, EventId, EventInfo, Globals, Init, Storage}; use crate::signal::RxFuture; use crate::sync::watch; use mio::net::UnixStream; use std::io::{self, Error, ErrorKind, Write}; -use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Once; use std::task::{Context, Poll}; -pub(crate) mod driver; -use self::driver::Handle; - pub(crate) type OsStorage = Vec<SignalInfo>; -// Number of different unix signals -// (FreeBSD has 33) -const SIGNUM: usize = 33; - impl Init for OsStorage { fn init() -> Self { - (0..SIGNUM).map(|_| SignalInfo::default()).collect() + // There are reliable signals ranging from 1 to 33 available on every Unix platform. + #[cfg(not(target_os = "linux"))] + let possible = 0..=33; + + // On Linux, there are additional real-time signals available. + #[cfg(target_os = "linux")] + let possible = 0..=libc::SIGRTMAX(); + + possible.map(|_| SignalInfo::default()).collect() } } @@ -48,7 +50,7 @@ impl Storage for OsStorage { #[derive(Debug)] pub(crate) struct OsExtraData { sender: UnixStream, - receiver: UnixStream, + pub(crate) receiver: UnixStream, } impl Init for OsExtraData { @@ -60,7 +62,7 @@ impl Init for OsExtraData { } /// Represents the specific kind of signal to listen for. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] pub struct SignalKind(libc::c_int); impl SignalKind { @@ -80,15 +82,26 @@ impl SignalKind { // unlikely to change to other types, but technically libc can change this // in the future minor version. // See https://github.com/tokio-rs/tokio/issues/3767 for more. - pub fn from_raw(signum: std::os::raw::c_int) -> Self { + pub const fn from_raw(signum: std::os::raw::c_int) -> Self { Self(signum as libc::c_int) } + /// Get the signal's numeric value. + /// + /// ```rust + /// # use tokio::signal::unix::SignalKind; + /// let kind = SignalKind::interrupt(); + /// assert_eq!(kind.as_raw_value(), libc::SIGINT); + /// ``` + pub const fn as_raw_value(&self) -> std::os::raw::c_int { + self.0 + } + /// Represents the SIGALRM signal. /// /// On Unix systems this signal is sent when a real-time timer has expired. /// By default, the process is terminated by this signal. - pub fn alarm() -> Self { + pub const fn alarm() -> Self { Self(libc::SIGALRM) } @@ -96,7 +109,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent when the status of a child process /// has changed. By default, this signal is ignored. - pub fn child() -> Self { + pub const fn child() -> Self { Self(libc::SIGCHLD) } @@ -104,7 +117,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent when the terminal is disconnected. /// By default, the process is terminated by this signal. - pub fn hangup() -> Self { + pub const fn hangup() -> Self { Self(libc::SIGHUP) } @@ -119,7 +132,7 @@ impl SignalKind { target_os = "netbsd", target_os = "openbsd" ))] - pub fn info() -> Self { + pub const fn info() -> Self { Self(libc::SIGINFO) } @@ -127,7 +140,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent to interrupt a program. /// By default, the process is terminated by this signal. - pub fn interrupt() -> Self { + pub const fn interrupt() -> Self { Self(libc::SIGINT) } @@ -135,7 +148,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent when I/O operations are possible /// on some file descriptor. By default, this signal is ignored. - pub fn io() -> Self { + pub const fn io() -> Self { Self(libc::SIGIO) } @@ -144,7 +157,7 @@ impl SignalKind { /// On Unix systems this signal is sent when the process attempts to write /// to a pipe which has no reader. By default, the process is terminated by /// this signal. - pub fn pipe() -> Self { + pub const fn pipe() -> Self { Self(libc::SIGPIPE) } @@ -153,7 +166,7 @@ impl SignalKind { /// On Unix systems this signal is sent to issue a shutdown of the /// process, after which the OS will dump the process core. /// By default, the process is terminated by this signal. - pub fn quit() -> Self { + pub const fn quit() -> Self { Self(libc::SIGQUIT) } @@ -161,7 +174,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent to issue a shutdown of the /// process. By default, the process is terminated by this signal. - pub fn terminate() -> Self { + pub const fn terminate() -> Self { Self(libc::SIGTERM) } @@ -169,7 +182,7 @@ impl SignalKind { /// /// On Unix systems this is a user defined signal. /// By default, the process is terminated by this signal. - pub fn user_defined1() -> Self { + pub const fn user_defined1() -> Self { Self(libc::SIGUSR1) } @@ -177,7 +190,7 @@ impl SignalKind { /// /// On Unix systems this is a user defined signal. /// By default, the process is terminated by this signal. - pub fn user_defined2() -> Self { + pub const fn user_defined2() -> Self { Self(libc::SIGUSR2) } @@ -185,11 +198,23 @@ impl SignalKind { /// /// On Unix systems this signal is sent when the terminal window is resized. /// By default, this signal is ignored. - pub fn window_change() -> Self { + pub const fn window_change() -> Self { Self(libc::SIGWINCH) } } +impl From<std::os::raw::c_int> for SignalKind { + fn from(signum: std::os::raw::c_int) -> Self { + Self::from_raw(signum as libc::c_int) + } +} + +impl From<SignalKind> for std::os::raw::c_int { + fn from(kind: SignalKind) -> Self { + kind.as_raw_value() + } +} + pub(crate) struct SignalInfo { event_info: EventInfo, init: Once, @@ -214,7 +239,7 @@ impl Default for SignalInfo { /// 2. Wake up the driver by writing a byte to a pipe /// /// Those two operations should both be async-signal safe. -fn action(globals: Pin<&'static Globals>, signal: libc::c_int) { +fn action(globals: &'static Globals, signal: libc::c_int) { globals.record_event(signal as EventId); // Send a wakeup, ignore any errors (anything reasonably possible is @@ -357,8 +382,15 @@ pub struct Signal { /// * If the previous initialization of this specific signal failed. /// * If the signal is one of /// [`signal_hook::FORBIDDEN`](fn@signal_hook_registry::register#panics) +/// +/// # Panics +/// +/// This function panics if there is no current reactor set, or if the `rt` +/// feature flag is not enabled. +#[track_caller] pub fn signal(kind: SignalKind) -> io::Result<Signal> { - let rx = signal_with_handle(kind, &Handle::current())?; + let handle = scheduler::Handle::current(); + let rx = signal_with_handle(kind, handle.driver().signal())?; Ok(Signal { inner: RxFuture::new(rx), @@ -380,6 +412,12 @@ impl Signal { /// /// `None` is returned if no more events can be received by this stream. /// + /// # Cancel safety + /// + /// This method is cancel safe. If you use it as the event in a + /// [`tokio::select!`](crate::select) statement and some other branch + /// completes first, then it is guaranteed that no signal is lost. + /// /// # Examples /// /// Wait for SIGHUP @@ -474,4 +512,15 @@ mod tests { ) .unwrap_err(); } + + #[test] + fn from_c_int() { + assert_eq!(SignalKind::from(2), SignalKind::interrupt()); + } + + #[test] + fn into_c_int() { + let value: std::os::raw::c_int = SignalKind::interrupt().into(); + assert_eq!(value, libc::SIGINT as _); + } } diff --git a/src/signal/unix/driver.rs b/src/signal/unix/driver.rs deleted file mode 100644 index 5fe7c35..0000000 --- a/src/signal/unix/driver.rs +++ /dev/null @@ -1,207 +0,0 @@ -#![cfg_attr(not(feature = "rt"), allow(dead_code))] - -//! Signal driver - -use crate::io::driver::{Driver as IoDriver, Interest}; -use crate::io::PollEvented; -use crate::park::Park; -use crate::signal::registry::globals; - -use mio::net::UnixStream; -use std::io::{self, Read}; -use std::ptr; -use std::sync::{Arc, Weak}; -use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; -use std::time::Duration; - -/// Responsible for registering wakeups when an OS signal is received, and -/// subsequently dispatching notifications to any signal listeners as appropriate. -/// -/// Note: this driver relies on having an enabled IO driver in order to listen to -/// pipe write wakeups. -#[derive(Debug)] -pub(crate) struct Driver { - /// Thread parker. The `Driver` park implementation delegates to this. - park: IoDriver, - - /// A pipe for receiving wake events from the signal handler - receiver: PollEvented<UnixStream>, - - /// Shared state - inner: Arc<Inner>, -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct Handle { - inner: Weak<Inner>, -} - -#[derive(Debug)] -pub(super) struct Inner(()); - -// ===== impl Driver ===== - -impl Driver { - /// Creates a new signal `Driver` instance that delegates wakeups to `park`. - pub(crate) fn new(park: IoDriver) -> io::Result<Self> { - use std::mem::ManuallyDrop; - use std::os::unix::io::{AsRawFd, FromRawFd}; - - // NB: We give each driver a "fresh" receiver file descriptor to avoid - // the issues described in alexcrichton/tokio-process#42. - // - // In the past we would reuse the actual receiver file descriptor and - // swallow any errors around double registration of the same descriptor. - // I'm not sure if the second (failed) registration simply doesn't end - // up receiving wake up notifications, or there could be some race - // condition when consuming readiness events, but having distinct - // descriptors for distinct PollEvented instances appears to mitigate - // this. - // - // Unfortunately we cannot just use a single global PollEvented instance - // either, since we can't compare Handles or assume they will always - // point to the exact same reactor. - // - // Mio 0.7 removed `try_clone()` as an API due to unexpected behavior - // with registering dups with the same reactor. In this case, duping is - // safe as each dup is registered with separate reactors **and** we - // only expect at least one dup to receive the notification. - - // Manually drop as we don't actually own this instance of UnixStream. - let receiver_fd = globals().receiver.as_raw_fd(); - - // safety: there is nothing unsafe about this, but the `from_raw_fd` fn is marked as unsafe. - let original = - ManuallyDrop::new(unsafe { std::os::unix::net::UnixStream::from_raw_fd(receiver_fd) }); - let receiver = UnixStream::from_std(original.try_clone()?); - let receiver = PollEvented::new_with_interest_and_handle( - receiver, - Interest::READABLE | Interest::WRITABLE, - park.handle(), - )?; - - Ok(Self { - park, - receiver, - inner: Arc::new(Inner(())), - }) - } - - /// Returns a handle to this event loop which can be sent across threads - /// and can be used as a proxy to the event loop itself. - pub(crate) fn handle(&self) -> Handle { - Handle { - inner: Arc::downgrade(&self.inner), - } - } - - fn process(&self) { - // Check if the pipe is ready to read and therefore has "woken" us up - // - // To do so, we will `poll_read_ready` with a noop waker, since we don't - // need to actually be notified when read ready... - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &NOOP_WAKER_VTABLE)) }; - let mut cx = Context::from_waker(&waker); - - let ev = match self.receiver.registration().poll_read_ready(&mut cx) { - Poll::Ready(Ok(ev)) => ev, - Poll::Ready(Err(e)) => panic!("reactor gone: {}", e), - Poll::Pending => return, // No wake has arrived, bail - }; - - // Drain the pipe completely so we can receive a new readiness event - // if another signal has come in. - let mut buf = [0; 128]; - loop { - match (&*self.receiver).read(&mut buf) { - Ok(0) => panic!("EOF on self-pipe"), - Ok(_) => continue, // Keep reading - Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, - Err(e) => panic!("Bad read on self-pipe: {}", e), - } - } - - self.receiver.registration().clear_readiness(ev); - - // Broadcast any signals which were received - globals().broadcast(); - } -} - -const NOOP_WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new(noop_clone, noop, noop, noop); - -unsafe fn noop_clone(_data: *const ()) -> RawWaker { - RawWaker::new(ptr::null(), &NOOP_WAKER_VTABLE) -} - -unsafe fn noop(_data: *const ()) {} - -// ===== impl Park for Driver ===== - -impl Park for Driver { - type Unpark = <IoDriver as Park>::Unpark; - type Error = io::Error; - - fn unpark(&self) -> Self::Unpark { - self.park.unpark() - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.park.park()?; - self.process(); - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.park.park_timeout(duration)?; - self.process(); - Ok(()) - } - - fn shutdown(&mut self) { - self.park.shutdown() - } -} - -// ===== impl Handle ===== - -impl Handle { - pub(super) fn check_inner(&self) -> io::Result<()> { - if self.inner.strong_count() > 0 { - Ok(()) - } else { - Err(io::Error::new(io::ErrorKind::Other, "signal driver gone")) - } - } -} - -cfg_rt! { - impl Handle { - /// Returns a handle to the current driver - /// - /// # Panics - /// - /// This function panics if there is no current signal driver set. - pub(super) fn current() -> Self { - crate::runtime::context::signal_handle().expect( - "there is no signal driver running, must be called from the context of Tokio runtime", - ) - } - } -} - -cfg_not_rt! { - impl Handle { - /// Returns a handle to the current driver - /// - /// # Panics - /// - /// This function panics if there is no current signal driver set. - pub(super) fn current() -> Self { - panic!( - "there is no signal driver running, must be called from the context of Tokio runtime or with\ - `rt` enabled.", - ) - } - } -} diff --git a/src/signal/windows.rs b/src/signal/windows.rs index 11ec6cb..730f95d 100644 --- a/src/signal/windows.rs +++ b/src/signal/windows.rs @@ -1,9 +1,9 @@ //! Windows-specific types for signal handling. //! -//! This module is only defined on Windows and allows receiving "ctrl-c" -//! and "ctrl-break" notifications. These events are listened for via the -//! `SetConsoleCtrlHandler` function which receives events of the type -//! `CTRL_C_EVENT` and `CTRL_BREAK_EVENT`. +//! This module is only defined on Windows and allows receiving "ctrl-c", +//! "ctrl-break", "ctrl-logoff", "ctrl-shutdown", and "ctrl-close" +//! notifications. These events are listened for via the `SetConsoleCtrlHandler` +//! function which receives the corresponding windows_sys event type. #![cfg(any(windows, docsrs))] #![cfg_attr(docsrs, doc(cfg(all(windows, feature = "signal"))))] @@ -221,3 +221,297 @@ pub fn ctrl_break() -> io::Result<CtrlBreak> { inner: self::imp::ctrl_break()?, }) } + +/// Creates a new stream which receives "ctrl-close" notifications sent to the +/// process. +/// +/// # Examples +/// +/// ```rust,no_run +/// use tokio::signal::windows::ctrl_close; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box<dyn std::error::Error>> { +/// // An infinite stream of CTRL-CLOSE events. +/// let mut stream = ctrl_close()?; +/// +/// // Print whenever a CTRL-CLOSE event is received. +/// for countdown in (0..3).rev() { +/// stream.recv().await; +/// println!("got CTRL-CLOSE. {} more to exit", countdown); +/// } +/// +/// Ok(()) +/// } +/// ``` +pub fn ctrl_close() -> io::Result<CtrlClose> { + Ok(CtrlClose { + inner: self::imp::ctrl_close()?, + }) +} + +/// Represents a stream which receives "ctrl-close" notitifications sent to the process +/// via 'SetConsoleCtrlHandler'. +/// +/// A notification to this process notifies *all* streams listening for +/// this event. Moreover, the notifications **are coalesced** if they aren't processed +/// quickly enough. This means that if two notifications are received back-to-back, +/// then the stream may only receive one item about the two notifications. +#[must_use = "streams do nothing unless polled"] +#[derive(Debug)] +pub struct CtrlClose { + inner: RxFuture, +} + +impl CtrlClose { + /// Receives the next signal notification event. + /// + /// `None` is returned if no more events can be received by this stream. + /// + /// # Examples + /// + /// ```rust,no_run + /// use tokio::signal::windows::ctrl_close; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box<dyn std::error::Error>> { + /// // An infinite stream of CTRL-CLOSE events. + /// let mut stream = ctrl_close()?; + /// + /// // Print whenever a CTRL-CLOSE event is received. + /// stream.recv().await; + /// println!("got CTRL-CLOSE. Cleaning up before exiting"); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv(&mut self) -> Option<()> { + self.inner.recv().await + } + + /// Polls to receive the next signal notification event, outside of an + /// `async` context. + /// + /// `None` is returned if no more events can be received by this stream. + /// + /// # Examples + /// + /// Polling from a manually implemented future + /// + /// ```rust,no_run + /// use std::pin::Pin; + /// use std::future::Future; + /// use std::task::{Context, Poll}; + /// use tokio::signal::windows::CtrlClose; + /// + /// struct MyFuture { + /// ctrl_close: CtrlClose, + /// } + /// + /// impl Future for MyFuture { + /// type Output = Option<()>; + /// + /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + /// println!("polling MyFuture"); + /// self.ctrl_close.poll_recv(cx) + /// } + /// } + /// ``` + pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<()>> { + self.inner.poll_recv(cx) + } +} + +/// Creates a new stream which receives "ctrl-shutdown" notifications sent to the +/// process. +/// +/// # Examples +/// +/// ```rust,no_run +/// use tokio::signal::windows::ctrl_shutdown; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box<dyn std::error::Error>> { +/// // An infinite stream of CTRL-SHUTDOWN events. +/// let mut stream = ctrl_shutdown()?; +/// +/// stream.recv().await; +/// println!("got CTRL-SHUTDOWN. Cleaning up before exiting"); +/// +/// Ok(()) +/// } +/// ``` +pub fn ctrl_shutdown() -> io::Result<CtrlShutdown> { + Ok(CtrlShutdown { + inner: self::imp::ctrl_shutdown()?, + }) +} + +/// Represents a stream which receives "ctrl-shutdown" notitifications sent to the process +/// via 'SetConsoleCtrlHandler'. +/// +/// A notification to this process notifies *all* streams listening for +/// this event. Moreover, the notifications **are coalesced** if they aren't processed +/// quickly enough. This means that if two notifications are received back-to-back, +/// then the stream may only receive one item about the two notifications. +#[must_use = "streams do nothing unless polled"] +#[derive(Debug)] +pub struct CtrlShutdown { + inner: RxFuture, +} + +impl CtrlShutdown { + /// Receives the next signal notification event. + /// + /// `None` is returned if no more events can be received by this stream. + /// + /// # Examples + /// + /// ```rust,no_run + /// use tokio::signal::windows::ctrl_shutdown; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box<dyn std::error::Error>> { + /// // An infinite stream of CTRL-SHUTDOWN events. + /// let mut stream = ctrl_shutdown()?; + /// + /// // Print whenever a CTRL-SHUTDOWN event is received. + /// stream.recv().await; + /// println!("got CTRL-SHUTDOWN. Cleaning up before exiting"); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv(&mut self) -> Option<()> { + self.inner.recv().await + } + + /// Polls to receive the next signal notification event, outside of an + /// `async` context. + /// + /// `None` is returned if no more events can be received by this stream. + /// + /// # Examples + /// + /// Polling from a manually implemented future + /// + /// ```rust,no_run + /// use std::pin::Pin; + /// use std::future::Future; + /// use std::task::{Context, Poll}; + /// use tokio::signal::windows::CtrlShutdown; + /// + /// struct MyFuture { + /// ctrl_shutdown: CtrlShutdown, + /// } + /// + /// impl Future for MyFuture { + /// type Output = Option<()>; + /// + /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + /// println!("polling MyFuture"); + /// self.ctrl_shutdown.poll_recv(cx) + /// } + /// } + /// ``` + pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<()>> { + self.inner.poll_recv(cx) + } +} + +/// Creates a new stream which receives "ctrl-logoff" notifications sent to the +/// process. +/// +/// # Examples +/// +/// ```rust,no_run +/// use tokio::signal::windows::ctrl_logoff; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box<dyn std::error::Error>> { +/// // An infinite stream of CTRL-LOGOFF events. +/// let mut stream = ctrl_logoff()?; +/// +/// stream.recv().await; +/// println!("got CTRL-LOGOFF. Cleaning up before exiting"); +/// +/// Ok(()) +/// } +/// ``` +pub fn ctrl_logoff() -> io::Result<CtrlLogoff> { + Ok(CtrlLogoff { + inner: self::imp::ctrl_logoff()?, + }) +} + +/// Represents a stream which receives "ctrl-logoff" notitifications sent to the process +/// via 'SetConsoleCtrlHandler'. +/// +/// A notification to this process notifies *all* streams listening for +/// this event. Moreover, the notifications **are coalesced** if they aren't processed +/// quickly enough. This means that if two notifications are received back-to-back, +/// then the stream may only receive one item about the two notifications. +#[must_use = "streams do nothing unless polled"] +#[derive(Debug)] +pub struct CtrlLogoff { + inner: RxFuture, +} + +impl CtrlLogoff { + /// Receives the next signal notification event. + /// + /// `None` is returned if no more events can be received by this stream. + /// + /// # Examples + /// + /// ```rust,no_run + /// use tokio::signal::windows::ctrl_logoff; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box<dyn std::error::Error>> { + /// // An infinite stream of CTRL-LOGOFF events. + /// let mut stream = ctrl_logoff()?; + /// + /// // Print whenever a CTRL-LOGOFF event is received. + /// stream.recv().await; + /// println!("got CTRL-LOGOFF. Cleaning up before exiting"); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv(&mut self) -> Option<()> { + self.inner.recv().await + } + + /// Polls to receive the next signal notification event, outside of an + /// `async` context. + /// + /// `None` is returned if no more events can be received by this stream. + /// + /// # Examples + /// + /// Polling from a manually implemented future + /// + /// ```rust,no_run + /// use std::pin::Pin; + /// use std::future::Future; + /// use std::task::{Context, Poll}; + /// use tokio::signal::windows::CtrlLogoff; + /// + /// struct MyFuture { + /// ctrl_logoff: CtrlLogoff, + /// } + /// + /// impl Future for MyFuture { + /// type Output = Option<()>; + /// + /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + /// println!("polling MyFuture"); + /// self.ctrl_logoff.poll_recv(cx) + /// } + /// } + /// ``` + pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<()>> { + self.inner.poll_recv(cx) + } +} diff --git a/src/signal/windows/stub.rs b/src/signal/windows/stub.rs index 8863054..61df309 100644 --- a/src/signal/windows/stub.rs +++ b/src/signal/windows/stub.rs @@ -4,10 +4,22 @@ use crate::signal::RxFuture; use std::io; +pub(super) fn ctrl_break() -> io::Result<RxFuture> { + panic!() +} + +pub(super) fn ctrl_close() -> io::Result<RxFuture> { + panic!() +} + pub(super) fn ctrl_c() -> io::Result<RxFuture> { panic!() } -pub(super) fn ctrl_break() -> io::Result<RxFuture> { +pub(super) fn ctrl_logoff() -> io::Result<RxFuture> { + panic!() +} + +pub(super) fn ctrl_shutdown() -> io::Result<RxFuture> { panic!() } diff --git a/src/signal/windows/sys.rs b/src/signal/windows/sys.rs index 8d29c35..f8133e0 100644 --- a/src/signal/windows/sys.rs +++ b/src/signal/windows/sys.rs @@ -5,19 +5,30 @@ use std::sync::Once; use crate::signal::registry::{globals, EventId, EventInfo, Init, Storage}; use crate::signal::RxFuture; -use winapi::shared::minwindef::{BOOL, DWORD, FALSE, TRUE}; -use winapi::um::consoleapi::SetConsoleCtrlHandler; -use winapi::um::wincon::{CTRL_BREAK_EVENT, CTRL_C_EVENT}; +use windows_sys::Win32::Foundation::BOOL; +use windows_sys::Win32::System::Console as console; + +pub(super) fn ctrl_break() -> io::Result<RxFuture> { + new(console::CTRL_BREAK_EVENT) +} + +pub(super) fn ctrl_close() -> io::Result<RxFuture> { + new(console::CTRL_CLOSE_EVENT) +} pub(super) fn ctrl_c() -> io::Result<RxFuture> { - new(CTRL_C_EVENT) + new(console::CTRL_C_EVENT) } -pub(super) fn ctrl_break() -> io::Result<RxFuture> { - new(CTRL_BREAK_EVENT) +pub(super) fn ctrl_logoff() -> io::Result<RxFuture> { + new(console::CTRL_LOGOFF_EVENT) +} + +pub(super) fn ctrl_shutdown() -> io::Result<RxFuture> { + new(console::CTRL_SHUTDOWN_EVENT) } -fn new(signum: DWORD) -> io::Result<RxFuture> { +fn new(signum: u32) -> io::Result<RxFuture> { global_init()?; let rx = globals().register_listener(signum as EventId); Ok(RxFuture::new(rx)) @@ -25,24 +36,33 @@ fn new(signum: DWORD) -> io::Result<RxFuture> { #[derive(Debug)] pub(crate) struct OsStorage { - ctrl_c: EventInfo, ctrl_break: EventInfo, + ctrl_close: EventInfo, + ctrl_c: EventInfo, + ctrl_logoff: EventInfo, + ctrl_shutdown: EventInfo, } impl Init for OsStorage { fn init() -> Self { Self { - ctrl_c: EventInfo::default(), - ctrl_break: EventInfo::default(), + ctrl_break: Default::default(), + ctrl_close: Default::default(), + ctrl_c: Default::default(), + ctrl_logoff: Default::default(), + ctrl_shutdown: Default::default(), } } } impl Storage for OsStorage { fn event_info(&self, id: EventId) -> Option<&EventInfo> { - match DWORD::try_from(id) { - Ok(CTRL_C_EVENT) => Some(&self.ctrl_c), - Ok(CTRL_BREAK_EVENT) => Some(&self.ctrl_break), + match u32::try_from(id) { + Ok(console::CTRL_BREAK_EVENT) => Some(&self.ctrl_break), + Ok(console::CTRL_CLOSE_EVENT) => Some(&self.ctrl_close), + Ok(console::CTRL_C_EVENT) => Some(&self.ctrl_c), + Ok(console::CTRL_LOGOFF_EVENT) => Some(&self.ctrl_logoff), + Ok(console::CTRL_SHUTDOWN_EVENT) => Some(&self.ctrl_shutdown), _ => None, } } @@ -51,8 +71,11 @@ impl Storage for OsStorage { where F: FnMut(&'a EventInfo), { - f(&self.ctrl_c); f(&self.ctrl_break); + f(&self.ctrl_close); + f(&self.ctrl_c); + f(&self.ctrl_logoff); + f(&self.ctrl_shutdown); } } @@ -71,7 +94,7 @@ fn global_init() -> io::Result<()> { let mut init = None; INIT.call_once(|| unsafe { - let rc = SetConsoleCtrlHandler(Some(handler), TRUE); + let rc = console::SetConsoleCtrlHandler(Some(handler), 1); let ret = if rc == 0 { Err(io::Error::last_os_error()) } else { @@ -84,7 +107,7 @@ fn global_init() -> io::Result<()> { init.unwrap_or_else(|| Ok(())) } -unsafe extern "system" fn handler(ty: DWORD) -> BOOL { +unsafe extern "system" fn handler(ty: u32) -> BOOL { let globals = globals(); globals.record_event(ty as EventId); @@ -93,11 +116,11 @@ unsafe extern "system" fn handler(ty: DWORD) -> BOOL { // have the same restrictions as in Unix signal handlers, meaning we can // go ahead and perform the broadcast here. if globals.broadcast() { - TRUE + 1 } else { // No one is listening for this notification any more // let the OS fire the next (possibly the default) handler. - FALSE + 0 } } @@ -121,7 +144,7 @@ mod tests { // like sending signals on Unix, so we'll stub out the actual OS // integration and test that our handling works. unsafe { - super::handler(CTRL_C_EVENT); + super::handler(console::CTRL_C_EVENT); } assert_ready_ok!(ctrl_c.poll()); @@ -138,13 +161,67 @@ mod tests { // like sending signals on Unix, so we'll stub out the actual OS // integration and test that our handling works. unsafe { - super::handler(CTRL_BREAK_EVENT); + super::handler(console::CTRL_BREAK_EVENT); } ctrl_break.recv().await.unwrap(); }); } + #[test] + fn ctrl_close() { + let rt = rt(); + + rt.block_on(async { + let mut ctrl_close = assert_ok!(crate::signal::windows::ctrl_close()); + + // Windows doesn't have a good programmatic way of sending events + // like sending signals on Unix, so we'll stub out the actual OS + // integration and test that our handling works. + unsafe { + super::handler(console::CTRL_CLOSE_EVENT); + } + + ctrl_close.recv().await.unwrap(); + }); + } + + #[test] + fn ctrl_shutdown() { + let rt = rt(); + + rt.block_on(async { + let mut ctrl_shutdown = assert_ok!(crate::signal::windows::ctrl_shutdown()); + + // Windows doesn't have a good programmatic way of sending events + // like sending signals on Unix, so we'll stub out the actual OS + // integration and test that our handling works. + unsafe { + super::handler(console::CTRL_SHUTDOWN_EVENT); + } + + ctrl_shutdown.recv().await.unwrap(); + }); + } + + #[test] + fn ctrl_logoff() { + let rt = rt(); + + rt.block_on(async { + let mut ctrl_logoff = assert_ok!(crate::signal::windows::ctrl_logoff()); + + // Windows doesn't have a good programmatic way of sending events + // like sending signals on Unix, so we'll stub out the actual OS + // integration and test that our handling works. + unsafe { + super::handler(console::CTRL_LOGOFF_EVENT); + } + + ctrl_logoff.recv().await.unwrap(); + }); + } + fn rt() -> Runtime { crate::runtime::Builder::new_current_thread() .build() diff --git a/src/sync/barrier.rs b/src/sync/barrier.rs index 0e39dac..2ce1d73 100644 --- a/src/sync/barrier.rs +++ b/src/sync/barrier.rs @@ -1,5 +1,7 @@ use crate::loom::sync::Mutex; use crate::sync::watch; +#[cfg(all(tokio_unstable, feature = "tracing"))] +use crate::util::trace; /// A barrier enables multiple tasks to synchronize the beginning of some computation. /// @@ -41,6 +43,8 @@ pub struct Barrier { state: Mutex<BarrierState>, wait: watch::Receiver<usize>, n: usize, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, } #[derive(Debug)] @@ -55,6 +59,7 @@ impl Barrier { /// /// A barrier will block `n`-1 tasks which call [`Barrier::wait`] and then wake up all /// tasks at once when the `n`th task calls `wait`. + #[track_caller] pub fn new(mut n: usize) -> Barrier { let (waker, wait) = crate::sync::watch::channel(0); @@ -65,6 +70,32 @@ impl Barrier { n = 1; } + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let location = std::panic::Location::caller(); + let resource_span = tracing::trace_span!( + "runtime.resource", + concrete_type = "Barrier", + kind = "Sync", + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + ); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + size = n, + ); + + tracing::trace!( + target: "runtime::resource::state_update", + arrived = 0, + ) + }); + resource_span + }; + Barrier { state: Mutex::new(BarrierState { waker, @@ -73,6 +104,8 @@ impl Barrier { }), n, wait, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -85,10 +118,24 @@ impl Barrier { /// [`BarrierWaitResult::is_leader`] when returning from this function, and all other tasks /// will receive a result that will return `false` from `is_leader`. pub async fn wait(&self) -> BarrierWaitResult { + #[cfg(all(tokio_unstable, feature = "tracing"))] + return trace::async_op( + || self.wait_internal(), + self.resource_span.clone(), + "Barrier::wait", + "poll", + false, + ) + .await; + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + return self.wait_internal().await; + } + async fn wait_internal(&self) -> BarrierWaitResult { // NOTE: we are taking a _synchronous_ lock here. // It is okay to do so because the critical section is fast and never yields, so it cannot // deadlock even if another future is concurrently holding the lock. - // It is _desireable_ to do so as synchronous Mutexes are, at least in theory, faster than + // It is _desirable_ to do so as synchronous Mutexes are, at least in theory, faster than // the asynchronous counter-parts, so we should use them where possible [citation needed]. // NOTE: the extra scope here is so that the compiler doesn't think `state` is held across // a yield point, and thus marks the returned future as !Send. @@ -96,7 +143,23 @@ impl Barrier { let mut state = self.state.lock(); let generation = state.generation; state.arrived += 1; + #[cfg(all(tokio_unstable, feature = "tracing"))] + tracing::trace!( + target: "runtime::resource::state_update", + arrived = 1, + arrived.op = "add", + ); + #[cfg(all(tokio_unstable, feature = "tracing"))] + tracing::trace!( + target: "runtime::resource::async_op::state_update", + arrived = true, + ); if state.arrived == self.n { + #[cfg(all(tokio_unstable, feature = "tracing"))] + tracing::trace!( + target: "runtime::resource::async_op::state_update", + is_leader = true, + ); // we are the leader for this generation // wake everyone, increment the generation, and return state diff --git a/src/sync/batch_semaphore.rs b/src/sync/batch_semaphore.rs index b5c39d2..57493f4 100644 --- a/src/sync/batch_semaphore.rs +++ b/src/sync/batch_semaphore.rs @@ -19,6 +19,8 @@ use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::{Mutex, MutexGuard}; use crate::util::linked_list::{self, LinkedList}; +#[cfg(all(tokio_unstable, feature = "tracing"))] +use crate::util::trace; use crate::util::WakeList; use std::future::Future; @@ -35,6 +37,8 @@ pub(crate) struct Semaphore { waiters: Mutex<Waitlist>, /// The current number of available permits in the semaphore. permits: AtomicUsize, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, } struct Waitlist { @@ -45,7 +49,7 @@ struct Waitlist { /// Error returned from the [`Semaphore::try_acquire`] function. /// /// [`Semaphore::try_acquire`]: crate::sync::Semaphore::try_acquire -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum TryAcquireError { /// The semaphore has been [closed] and cannot issue new permits. /// @@ -101,10 +105,21 @@ struct Waiter { /// use `UnsafeCell` internally. pointers: linked_list::Pointers<Waiter>, + #[cfg(all(tokio_unstable, feature = "tracing"))] + ctx: trace::AsyncOpTracingCtx, + /// Should not be `Unpin`. _p: PhantomPinned, } +generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> { + &self.pointers + } + } +} + impl Semaphore { /// The maximum number of permits which a semaphore can hold. /// @@ -129,12 +144,34 @@ impl Semaphore { "a semaphore may not have more than MAX_PERMITS permits ({})", Self::MAX_PERMITS ); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let resource_span = tracing::trace_span!( + "runtime.resource", + concrete_type = "Semaphore", + kind = "Sync", + is_internal = true + ); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + permits = permits, + permits.op = "override", + ) + }); + resource_span + }; + Self { permits: AtomicUsize::new(permits << Self::PERMIT_SHIFT), waiters: Mutex::new(Waitlist { queue: LinkedList::new(), closed: false, }), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -156,6 +193,8 @@ impl Semaphore { queue: LinkedList::new(), closed: false, }), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span::none(), } } @@ -224,7 +263,10 @@ impl Semaphore { let next = curr - num_permits; match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { - Ok(_) => return Ok(()), + Ok(_) => { + // TODO: Instrument once issue has been solved} + return Ok(()); + } Err(actual) => curr = actual, } } @@ -283,6 +325,17 @@ impl Semaphore { rem, Self::MAX_PERMITS ); + + // add remaining permits back + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + permits = rem, + permits.op = "add", + ) + }); + rem = 0; } @@ -347,6 +400,20 @@ impl Semaphore { acquired += acq; if remaining == 0 { if !queued { + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + permits = acquired, + permits.op = "sub", + ); + tracing::trace!( + target: "runtime::resource::async_op::state_update", + permits_obtained = acquired, + permits.op = "add", + ) + }); + return Ready(Ok(())); } else if lock.is_none() { break self.waiters.lock(); @@ -362,6 +429,15 @@ impl Semaphore { return Ready(Err(AcquireError::closed())); } + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + permits = acquired, + permits.op = "sub", + ) + }); + if node.assign_permits(&mut acquired) { self.add_permits_locked(acquired, waiters); return Ready(Ok(())); @@ -406,11 +482,16 @@ impl fmt::Debug for Semaphore { } impl Waiter { - fn new(num_permits: u32) -> Self { + fn new( + num_permits: u32, + #[cfg(all(tokio_unstable, feature = "tracing"))] ctx: trace::AsyncOpTracingCtx, + ) -> Self { Waiter { waker: UnsafeCell::new(None), state: AtomicUsize::new(num_permits as usize), pointers: linked_list::Pointers::new(), + #[cfg(all(tokio_unstable, feature = "tracing"))] + ctx, _p: PhantomPinned, } } @@ -426,6 +507,14 @@ impl Waiter { match self.state.compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => { *n -= assign; + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.ctx.async_op_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::async_op::state_update", + permits_obtained = assign, + permits.op = "add", + ); + }); return next == 0; } Err(actual) => curr = actual, @@ -438,12 +527,26 @@ impl Future for Acquire<'_> { type Output = Result<(), AcquireError>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { - // First, ensure the current task has enough budget to proceed. - let coop = ready!(crate::coop::poll_proceed(cx)); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _resource_span = self.node.ctx.resource_span.clone().entered(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _async_op_span = self.node.ctx.async_op_span.clone().entered(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _async_op_poll_span = self.node.ctx.async_op_poll_span.clone().entered(); let (node, semaphore, needed, queued) = self.project(); - match semaphore.poll_acquire(cx, needed, node, *queued) { + // First, ensure the current task has enough budget to proceed. + #[cfg(all(tokio_unstable, feature = "tracing"))] + let coop = ready!(trace_poll_op!( + "poll_acquire", + crate::runtime::coop::poll_proceed(cx), + )); + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); + + let result = match semaphore.poll_acquire(cx, needed, node, *queued) { Pending => { *queued = true; Pending @@ -454,18 +557,59 @@ impl Future for Acquire<'_> { *queued = false; Ready(Ok(())) } - } + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + return trace_poll_op!("poll_acquire", result); + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + return result; } } impl<'a> Acquire<'a> { fn new(semaphore: &'a Semaphore, num_permits: u32) -> Self { - Self { + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + return Self { node: Waiter::new(num_permits), semaphore, num_permits, queued: false, - } + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + return semaphore.resource_span.in_scope(|| { + let async_op_span = + tracing::trace_span!("runtime.resource.async_op", source = "Acquire::new"); + let async_op_poll_span = async_op_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::async_op::state_update", + permits_requested = num_permits, + permits.op = "override", + ); + + tracing::trace!( + target: "runtime::resource::async_op::state_update", + permits_obtained = 0usize, + permits.op = "override", + ); + + tracing::trace_span!("runtime.resource.async_op.poll") + }); + + let ctx = trace::AsyncOpTracingCtx { + async_op_span, + async_op_poll_span, + resource_span: semaphore.resource_span.clone(), + }; + + Self { + node: Waiter::new(num_permits, ctx), + semaphore, + num_permits, + queued: false, + } + }); } fn project(self: Pin<&mut Self>) -> (Pin<&mut Waiter>, &Semaphore, u32, &mut bool) { @@ -568,12 +712,6 @@ impl std::error::Error for TryAcquireError {} /// /// `Waiter` is forced to be !Unpin. unsafe impl linked_list::Link for Waiter { - // XXX: ideally, we would be able to use `Pin` here, to enforce the - // invariant that list entries may not move while in the list. However, we - // can't do this currently, as using `Pin<&'a mut Waiter>` as the `Handle` - // type would require `Semaphore` to be generic over a lifetime. We can't - // use `Pin<*mut Waiter>`, as raw pointers are `Unpin` regardless of whether - // or not they dereference to an `!Unpin` target. type Handle = NonNull<Waiter>; type Target = Waiter; @@ -585,7 +723,7 @@ unsafe impl linked_list::Link for Waiter { ptr } - unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { + Waiter::addr_of_pointers(target) } } diff --git a/src/sync/broadcast.rs b/src/sync/broadcast.rs index 0d9cd3b..1c6b2ca 100644 --- a/src/sync/broadcast.rs +++ b/src/sync/broadcast.rs @@ -18,6 +18,9 @@ //! returned [`Receiver`] will receive values sent **after** the call to //! `subscribe`. //! +//! This channel is also suitable for the single-producer multi-consumer +//! use-case, where a single sender broadcasts values to many receivers. +//! //! ## Lagging //! //! As sent messages must be retained until **all** [`Receiver`] handles receive @@ -230,7 +233,7 @@ pub mod error { /// /// [`recv`]: crate::sync::broadcast::Receiver::recv /// [`Receiver`]: crate::sync::broadcast::Receiver - #[derive(Debug, PartialEq)] + #[derive(Debug, PartialEq, Eq, Clone)] pub enum RecvError { /// There are no more active senders implying no further messages will ever /// be sent. @@ -258,7 +261,7 @@ pub mod error { /// /// [`try_recv`]: crate::sync::broadcast::Receiver::try_recv /// [`Receiver`]: crate::sync::broadcast::Receiver - #[derive(Debug, PartialEq)] + #[derive(Debug, PartialEq, Eq, Clone)] pub enum TryRecvError { /// The channel is currently empty. There are still active /// [`Sender`] handles, so data may yet become available. @@ -336,9 +339,6 @@ struct Slot<T> { /// Uniquely identifies the `send` stored in the slot. pos: u64, - /// True signals the channel is closed. - closed: bool, - /// The value being broadcast. /// /// The value is set by `send` when the write lock is held. When a reader @@ -361,6 +361,14 @@ struct Waiter { _p: PhantomPinned, } +generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> { + &self.pointers + } + } +} + struct RecvGuard<'a, T> { slot: RwLockReadGuard<'a, Slot<T>>, } @@ -425,6 +433,12 @@ const MAX_RECEIVERS: usize = usize::MAX >> 2; /// tx.send(20).unwrap(); /// } /// ``` +/// +/// # Panics +/// +/// This will panic if `capacity` is equal to `0` or larger +/// than `usize::MAX / 2`. +#[track_caller] pub fn channel<T: Clone>(mut capacity: usize) -> (Sender<T>, Receiver<T>) { assert!(capacity > 0, "capacity is empty"); assert!(capacity <= usize::MAX >> 1, "requested capacity too large"); @@ -438,7 +452,6 @@ pub fn channel<T: Clone>(mut capacity: usize) -> (Sender<T>, Receiver<T>) { buffer.push(RwLock::new(Slot { rem: AtomicUsize::new(0), pos: (i as u64).wrapping_sub(capacity as u64), - closed: false, val: UnsafeCell::new(None), })); } @@ -523,8 +536,43 @@ impl<T> Sender<T> { /// } /// ``` pub fn send(&self, value: T) -> Result<usize, SendError<T>> { - self.send2(Some(value)) - .map_err(|SendError(maybe_v)| SendError(maybe_v.unwrap())) + let mut tail = self.shared.tail.lock(); + + if tail.rx_cnt == 0 { + return Err(SendError(value)); + } + + // Position to write into + let pos = tail.pos; + let rem = tail.rx_cnt; + let idx = (pos & self.shared.mask as u64) as usize; + + // Update the tail position + tail.pos = tail.pos.wrapping_add(1); + + // Get the slot + let mut slot = self.shared.buffer[idx].write().unwrap(); + + // Track the position + slot.pos = pos; + + // Set remaining receivers + slot.rem.with_mut(|v| *v = rem); + + // Write the value + slot.val = UnsafeCell::new(Some(value)); + + // Release the slot lock before notifying the receivers. + drop(slot); + + tail.notify_rx(); + + // Release the mutex. This must happen after the slot lock is released, + // otherwise the writer lock bit could be cleared while another thread + // is in the critical section. + drop(tail); + + Ok(rem) } /// Creates a new [`Receiver`] handle that will receive values sent **after** @@ -555,6 +603,97 @@ impl<T> Sender<T> { new_receiver(shared) } + /// Returns the number of queued values. + /// + /// A value is queued until it has either been seen by all receivers that were alive at the time + /// it was sent, or has been evicted from the queue by subsequent sends that exceeded the + /// queue's capacity. + /// + /// # Note + /// + /// In contrast to [`Receiver::len`], this method only reports queued values and not values that + /// have been evicted from the queue before being seen by all receivers. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// let mut rx2 = tx.subscribe(); + /// + /// tx.send(10).unwrap(); + /// tx.send(20).unwrap(); + /// tx.send(30).unwrap(); + /// + /// assert_eq!(tx.len(), 3); + /// + /// rx1.recv().await.unwrap(); + /// + /// // The len is still 3 since rx2 hasn't seen the first value yet. + /// assert_eq!(tx.len(), 3); + /// + /// rx2.recv().await.unwrap(); + /// + /// assert_eq!(tx.len(), 2); + /// } + /// ``` + pub fn len(&self) -> usize { + let tail = self.shared.tail.lock(); + + let base_idx = (tail.pos & self.shared.mask as u64) as usize; + let mut low = 0; + let mut high = self.shared.buffer.len(); + while low < high { + let mid = low + (high - low) / 2; + let idx = base_idx.wrapping_add(mid) & self.shared.mask; + if self.shared.buffer[idx].read().unwrap().rem.load(SeqCst) == 0 { + low = mid + 1; + } else { + high = mid; + } + } + + self.shared.buffer.len() - low + } + + /// Returns true if there are no queued values. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// let mut rx2 = tx.subscribe(); + /// + /// assert!(tx.is_empty()); + /// + /// tx.send(10).unwrap(); + /// + /// assert!(!tx.is_empty()); + /// + /// rx1.recv().await.unwrap(); + /// + /// // The queue is still not empty since rx2 hasn't seen the value. + /// assert!(!tx.is_empty()); + /// + /// rx2.recv().await.unwrap(); + /// + /// assert!(tx.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + let tail = self.shared.tail.lock(); + + let idx = (tail.pos.wrapping_sub(1) & self.shared.mask as u64) as usize; + self.shared.buffer[idx].read().unwrap().rem.load(SeqCst) == 0 + } + /// Returns the number of active receivers /// /// An active receiver is a [`Receiver`] handle returned from [`channel`] or @@ -596,52 +735,15 @@ impl<T> Sender<T> { tail.rx_cnt } - fn send2(&self, value: Option<T>) -> Result<usize, SendError<Option<T>>> { + fn close_channel(&self) { let mut tail = self.shared.tail.lock(); - - if tail.rx_cnt == 0 { - return Err(SendError(value)); - } - - // Position to write into - let pos = tail.pos; - let rem = tail.rx_cnt; - let idx = (pos & self.shared.mask as u64) as usize; - - // Update the tail position - tail.pos = tail.pos.wrapping_add(1); - - // Get the slot - let mut slot = self.shared.buffer[idx].write().unwrap(); - - // Track the position - slot.pos = pos; - - // Set remaining receivers - slot.rem.with_mut(|v| *v = rem); - - // Set the closed bit if the value is `None`; otherwise write the value - if value.is_none() { - tail.closed = true; - slot.closed = true; - } else { - slot.val.with_mut(|ptr| unsafe { *ptr = value }); - } - - // Release the slot lock before notifying the receivers. - drop(slot); + tail.closed = true; tail.notify_rx(); - - // Release the mutex. This must happen after the slot lock is released, - // otherwise the writer lock bit could be cleared while another thread - // is in the critical section. - drop(tail); - - Ok(rem) } } +/// Create a new `Receiver` which reads starting from the tail. fn new_receiver<T>(shared: Arc<Shared<T>>) -> Receiver<T> { let mut tail = shared.tail.lock(); @@ -685,12 +787,79 @@ impl<T> Clone for Sender<T> { impl<T> Drop for Sender<T> { fn drop(&mut self) { if 1 == self.shared.num_tx.fetch_sub(1, SeqCst) { - let _ = self.send2(None); + self.close_channel(); } } } impl<T> Receiver<T> { + /// Returns the number of messages that were sent into the channel and that + /// this [`Receiver`] has yet to receive. + /// + /// If the returned value from `len` is larger than the next largest power of 2 + /// of the capacity of the channel any call to [`recv`] will return an + /// `Err(RecvError::Lagged)` and any call to [`try_recv`] will return an + /// `Err(TryRecvError::Lagged)`, e.g. if the capacity of the channel is 10, + /// [`recv`] will start to return `Err(RecvError::Lagged)` once `len` returns + /// values larger than 16. + /// + /// [`Receiver`]: crate::sync::broadcast::Receiver + /// [`recv`]: crate::sync::broadcast::Receiver::recv + /// [`try_recv`]: crate::sync::broadcast::Receiver::try_recv + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// + /// tx.send(10).unwrap(); + /// tx.send(20).unwrap(); + /// + /// assert_eq!(rx1.len(), 2); + /// assert_eq!(rx1.recv().await.unwrap(), 10); + /// assert_eq!(rx1.len(), 1); + /// assert_eq!(rx1.recv().await.unwrap(), 20); + /// assert_eq!(rx1.len(), 0); + /// } + /// ``` + pub fn len(&self) -> usize { + let next_send_pos = self.shared.tail.lock().pos; + (next_send_pos - self.next) as usize + } + + /// Returns true if there aren't any messages in the channel that the [`Receiver`] + /// has yet to receive. + /// + /// [`Receiver]: create::sync::broadcast::Receiver + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// + /// assert!(rx1.is_empty()); + /// + /// tx.send(10).unwrap(); + /// tx.send(20).unwrap(); + /// + /// assert!(!rx1.is_empty()); + /// assert_eq!(rx1.recv().await.unwrap(), 10); + /// assert_eq!(rx1.recv().await.unwrap(), 20); + /// assert!(rx1.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Locks the next value if there is one. fn recv_ref( &mut self, @@ -702,14 +871,6 @@ impl<T> Receiver<T> { let mut slot = self.shared.buffer[idx].read().unwrap(); if slot.pos != self.next { - let next_pos = slot.pos.wrapping_add(self.shared.buffer.len() as u64); - - // The receiver has read all current values in the channel and there - // is no waiter to register - if waiter.is_none() && next_pos == self.next { - return Err(TryRecvError::Empty); - } - // Release the `slot` lock before attempting to acquire the `tail` // lock. This is required because `send2` acquires the tail lock // first followed by the slot lock. Acquiring the locks in reverse @@ -731,6 +892,13 @@ impl<T> Receiver<T> { let next_pos = slot.pos.wrapping_add(self.shared.buffer.len() as u64); if next_pos == self.next { + // At this point the channel is empty for *this* receiver. If + // it's been closed, then that's what we return, otherwise we + // set a waker and return empty. + if tail.closed { + return Err(TryRecvError::Closed); + } + // Store the waker if let Some((waiter, waker)) = waiter { // Safety: called while locked. @@ -764,22 +932,7 @@ impl<T> Receiver<T> { // catch up by skipping dropped messages and setting the // internal cursor to the **oldest** message stored by the // channel. - // - // However, finding the oldest position is a bit more - // complicated than `tail-position - buffer-size`. When - // the channel is closed, the tail position is incremented to - // signal a new `None` message, but `None` is not stored in the - // channel itself (see issue #2425 for why). - // - // To account for this, if the channel is closed, the tail - // position is decremented by `buffer-size + 1`. - let mut adjust = 0; - if tail.closed { - adjust = 1 - } - let next = tail - .pos - .wrapping_sub(self.shared.buffer.len() as u64 + adjust); + let next = tail.pos.wrapping_sub(self.shared.buffer.len() as u64); let missed = next.wrapping_sub(self.next); @@ -800,15 +953,38 @@ impl<T> Receiver<T> { self.next = self.next.wrapping_add(1); - if slot.closed { - return Err(TryRecvError::Closed); - } - Ok(RecvGuard { slot }) } } impl<T: Clone> Receiver<T> { + /// Re-subscribes to the channel starting from the current tail element. + /// + /// This [`Receiver`] handle will receive a clone of all values sent + /// **after** it has resubscribed. This will not include elements that are + /// in the queue of the current receiver. Consider the following example. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx) = broadcast::channel(2); + /// + /// tx.send(1).unwrap(); + /// let mut rx2 = rx.resubscribe(); + /// tx.send(2).unwrap(); + /// + /// assert_eq!(rx2.recv().await.unwrap(), 2); + /// assert_eq!(rx.recv().await.unwrap(), 1); + /// } + /// ``` + pub fn resubscribe(&self) -> Self { + let shared = self.shared.clone(); + new_receiver(shared) + } /// Receives the next value for this receiver. /// /// Each [`Receiver`] handle will receive a clone of all values sent @@ -1039,8 +1215,8 @@ unsafe impl linked_list::Link for Waiter { ptr } - unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { + Waiter::addr_of_pointers(target) } } diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 457e6ab..8fba196 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -94,6 +94,10 @@ //! producers to a single consumer. This channel is often used to send work to a //! task or to receive the result of many computations. //! +//! This is also the channel you should use if you want to send many messages +//! from a single producer to a single consumer. There is no dedicated spsc +//! channel. +//! //! **Example:** using an mpsc to incrementally stream the results of a series //! of computations. //! @@ -244,6 +248,10 @@ //! This channel tends to be used less often than `oneshot` and `mpsc` but still //! has its use cases. //! +//! This is also the channel you should use if you want to broadcast values from +//! a single producer to many consumers. There is no dedicated spmc broadcast +//! channel. +//! //! Basic usage //! //! ``` diff --git a/src/sync/mpsc/block.rs b/src/sync/mpsc/block.rs index 58f4a9f..39c3e1b 100644 --- a/src/sync/mpsc/block.rs +++ b/src/sync/mpsc/block.rs @@ -1,6 +1,7 @@ use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize}; +use std::alloc::Layout; use std::mem::MaybeUninit; use std::ops; use std::ptr::{self, NonNull}; @@ -10,6 +11,17 @@ use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Release}; /// /// Each block in the list can hold up to `BLOCK_CAP` messages. pub(crate) struct Block<T> { + /// The header fields. + header: BlockHeader<T>, + + /// Array containing values pushed into the block. Values are stored in a + /// continuous array in order to improve cache line behavior when reading. + /// The values must be manually dropped. + values: Values<T>, +} + +/// Extra fields for a `Block<T>`. +struct BlockHeader<T> { /// The start index of this block. /// /// Slots in this block have indices in `start_index .. start_index + BLOCK_CAP`. @@ -24,11 +36,6 @@ pub(crate) struct Block<T> { /// The observed `tail_position` value *after* the block has been passed by /// `block_tail`. observed_tail_position: UnsafeCell<usize>, - - /// Array containing values pushed into the block. Values are stored in a - /// continuous array in order to improve cache line behavior when reading. - /// The values must be manually dropped. - values: Values<T>, } pub(crate) enum Read<T> { @@ -36,6 +43,7 @@ pub(crate) enum Read<T> { Closed, } +#[repr(transparent)] struct Values<T>([UnsafeCell<MaybeUninit<T>>; BLOCK_CAP]); use super::BLOCK_CAP; @@ -71,28 +79,56 @@ pub(crate) fn offset(slot_index: usize) -> usize { SLOT_MASK & slot_index } +generate_addr_of_methods! { + impl<T> Block<T> { + unsafe fn addr_of_header(self: NonNull<Self>) -> NonNull<BlockHeader<T>> { + &self.header + } + + unsafe fn addr_of_values(self: NonNull<Self>) -> NonNull<Values<T>> { + &self.values + } + } +} + impl<T> Block<T> { - pub(crate) fn new(start_index: usize) -> Block<T> { - Block { - // The absolute index in the channel of the first slot in the block. - start_index, + pub(crate) fn new(start_index: usize) -> Box<Block<T>> { + unsafe { + // Allocate the block on the heap. + // SAFETY: The size of the Block<T> is non-zero, since it is at least the size of the header. + let block = std::alloc::alloc(Layout::new::<Block<T>>()) as *mut Block<T>; + let block = match NonNull::new(block) { + Some(block) => block, + None => std::alloc::handle_alloc_error(Layout::new::<Block<T>>()), + }; + + // Write the header to the block. + Block::addr_of_header(block).as_ptr().write(BlockHeader { + // The absolute index in the channel of the first slot in the block. + start_index, - // Pointer to the next block in the linked list. - next: AtomicPtr::new(ptr::null_mut()), + // Pointer to the next block in the linked list. + next: AtomicPtr::new(ptr::null_mut()), - ready_slots: AtomicUsize::new(0), + ready_slots: AtomicUsize::new(0), - observed_tail_position: UnsafeCell::new(0), + observed_tail_position: UnsafeCell::new(0), + }); - // Value storage - values: unsafe { Values::uninitialized() }, + // Initialize the values array. + Values::initialize(Block::addr_of_values(block)); + + // Convert the pointer to a `Box`. + // Safety: The raw pointer was allocated using the global allocator, and with + // the layout for a `Block<T>`, so it's valid to convert it to box. + Box::from_raw(block.as_ptr()) } } /// Returns `true` if the block matches the given index. pub(crate) fn is_at_index(&self, index: usize) -> bool { debug_assert!(offset(index) == 0); - self.start_index == index + self.header.start_index == index } /// Returns the number of blocks between `self` and the block at the @@ -101,7 +137,7 @@ impl<T> Block<T> { /// `start_index` must represent a block *after* `self`. pub(crate) fn distance(&self, other_index: usize) -> usize { debug_assert!(offset(other_index) == 0); - other_index.wrapping_sub(self.start_index) / BLOCK_CAP + other_index.wrapping_sub(self.header.start_index) / BLOCK_CAP } /// Reads the value at the given offset. @@ -116,7 +152,7 @@ impl<T> Block<T> { pub(crate) unsafe fn read(&self, slot_index: usize) -> Option<Read<T>> { let offset = offset(slot_index); - let ready_bits = self.ready_slots.load(Acquire); + let ready_bits = self.header.ready_slots.load(Acquire); if !is_ready(ready_bits, offset) { if is_tx_closed(ready_bits) { @@ -156,7 +192,7 @@ impl<T> Block<T> { /// Signal to the receiver that the sender half of the list is closed. pub(crate) unsafe fn tx_close(&self) { - self.ready_slots.fetch_or(TX_CLOSED, Release); + self.header.ready_slots.fetch_or(TX_CLOSED, Release); } /// Resets the block to a blank state. This enables reusing blocks in the @@ -169,9 +205,9 @@ impl<T> Block<T> { /// * All slots are empty. /// * The caller holds a unique pointer to the block. pub(crate) unsafe fn reclaim(&mut self) { - self.start_index = 0; - self.next = AtomicPtr::new(ptr::null_mut()); - self.ready_slots = AtomicUsize::new(0); + self.header.start_index = 0; + self.header.next = AtomicPtr::new(ptr::null_mut()); + self.header.ready_slots = AtomicUsize::new(0); } /// Releases the block to the rx half for freeing. @@ -187,19 +223,20 @@ impl<T> Block<T> { pub(crate) unsafe fn tx_release(&self, tail_position: usize) { // Track the observed tail_position. Any sender targeting a greater // tail_position is guaranteed to not access this block. - self.observed_tail_position + self.header + .observed_tail_position .with_mut(|ptr| *ptr = tail_position); // Set the released bit, signalling to the receiver that it is safe to // free the block's memory as soon as all slots **prior** to // `observed_tail_position` have been filled. - self.ready_slots.fetch_or(RELEASED, Release); + self.header.ready_slots.fetch_or(RELEASED, Release); } /// Mark a slot as ready fn set_ready(&self, slot: usize) { let mask = 1 << slot; - self.ready_slots.fetch_or(mask, Release); + self.header.ready_slots.fetch_or(mask, Release); } /// Returns `true` when all slots have their `ready` bits set. @@ -214,25 +251,31 @@ impl<T> Block<T> { /// single atomic cell. However, this could have negative impact on cache /// behavior as there would be many more mutations to a single slot. pub(crate) fn is_final(&self) -> bool { - self.ready_slots.load(Acquire) & READY_MASK == READY_MASK + self.header.ready_slots.load(Acquire) & READY_MASK == READY_MASK } /// Returns the `observed_tail_position` value, if set pub(crate) fn observed_tail_position(&self) -> Option<usize> { - if 0 == RELEASED & self.ready_slots.load(Acquire) { + if 0 == RELEASED & self.header.ready_slots.load(Acquire) { None } else { - Some(self.observed_tail_position.with(|ptr| unsafe { *ptr })) + Some( + self.header + .observed_tail_position + .with(|ptr| unsafe { *ptr }), + ) } } /// Loads the next block pub(crate) fn load_next(&self, ordering: Ordering) -> Option<NonNull<Block<T>>> { - let ret = NonNull::new(self.next.load(ordering)); + let ret = NonNull::new(self.header.next.load(ordering)); debug_assert!(unsafe { - ret.map(|block| block.as_ref().start_index == self.start_index.wrapping_add(BLOCK_CAP)) - .unwrap_or(true) + ret.map(|block| { + block.as_ref().header.start_index == self.header.start_index.wrapping_add(BLOCK_CAP) + }) + .unwrap_or(true) }); ret @@ -260,9 +303,10 @@ impl<T> Block<T> { success: Ordering, failure: Ordering, ) -> Result<(), NonNull<Block<T>>> { - block.as_mut().start_index = self.start_index.wrapping_add(BLOCK_CAP); + block.as_mut().header.start_index = self.header.start_index.wrapping_add(BLOCK_CAP); let next_ptr = self + .header .next .compare_exchange(ptr::null_mut(), block.as_ptr(), success, failure) .unwrap_or_else(|x| x); @@ -291,7 +335,7 @@ impl<T> Block<T> { // Create the new block. It is assumed that the block will become the // next one after `&self`. If this turns out to not be the case, // `start_index` is updated accordingly. - let new_block = Box::new(Block::new(self.start_index + BLOCK_CAP)); + let new_block = Block::new(self.header.start_index + BLOCK_CAP); let mut new_block = unsafe { NonNull::new_unchecked(Box::into_raw(new_block)) }; @@ -308,7 +352,8 @@ impl<T> Block<T> { // `Release` ensures that the newly allocated block is available to // other threads acquiring the next pointer. let next = NonNull::new( - self.next + self.header + .next .compare_exchange(ptr::null_mut(), new_block.as_ptr(), AcqRel, Acquire) .unwrap_or_else(|x| x), ); @@ -360,19 +405,20 @@ fn is_tx_closed(bits: usize) -> bool { } impl<T> Values<T> { - unsafe fn uninitialized() -> Values<T> { - let mut vals = MaybeUninit::uninit(); - + /// Initialize a `Values` struct from a pointer. + /// + /// # Safety + /// + /// The raw pointer must be valid for writing a `Values<T>`. + unsafe fn initialize(_value: NonNull<Values<T>>) { // When fuzzing, `UnsafeCell` needs to be initialized. if_loom! { - let p = vals.as_mut_ptr() as *mut UnsafeCell<MaybeUninit<T>>; + let p = _value.as_ptr() as *mut UnsafeCell<MaybeUninit<T>>; for i in 0..BLOCK_CAP { p.add(i) .write(UnsafeCell::new(MaybeUninit::uninit())); } } - - Values(vals.assume_init()) } } @@ -383,3 +429,20 @@ impl<T> ops::Index<usize> for Values<T> { self.0.index(index) } } + +#[cfg(all(test, not(loom)))] +#[test] +fn assert_no_stack_overflow() { + // https://github.com/tokio-rs/tokio/issues/5293 + + struct Foo { + _a: [u8; 2_000_000], + } + + assert_eq!( + Layout::new::<MaybeUninit<Block<Foo>>>(), + Layout::new::<Block<Foo>>() + ); + + let _block = Block::<Foo>::new(0); +} diff --git a/src/sync/mpsc/bounded.rs b/src/sync/mpsc/bounded.rs index 5a2bfa6..8babdc7 100644 --- a/src/sync/mpsc/bounded.rs +++ b/src/sync/mpsc/bounded.rs @@ -1,3 +1,4 @@ +use crate::loom::sync::Arc; use crate::sync::batch_semaphore::{self as semaphore, TryAcquireError}; use crate::sync::mpsc::chan; use crate::sync::mpsc::error::{SendError, TryRecvError, TrySendError}; @@ -22,6 +23,40 @@ pub struct Sender<T> { chan: chan::Tx<T, Semaphore>, } +/// A sender that does not prevent the channel from being closed. +/// +/// If all [`Sender`] instances of a channel were dropped and only `WeakSender` +/// instances remain, the channel is closed. +/// +/// In order to send messages, the `WeakSender` needs to be upgraded using +/// [`WeakSender::upgrade`], which returns `Option<Sender>`. It returns `None` +/// if all `Sender`s have been dropped, and otherwise it returns a `Sender`. +/// +/// [`Sender`]: Sender +/// [`WeakSender::upgrade`]: WeakSender::upgrade +/// +/// #Examples +/// +/// ``` +/// use tokio::sync::mpsc::channel; +/// +/// #[tokio::main] +/// async fn main() { +/// let (tx, _rx) = channel::<i32>(15); +/// let tx_weak = tx.downgrade(); +/// +/// // Upgrading will succeed because `tx` still exists. +/// assert!(tx_weak.upgrade().is_some()); +/// +/// // If we drop `tx`, then it will fail. +/// drop(tx); +/// assert!(tx_weak.clone().upgrade().is_none()); +/// } +/// ``` +pub struct WeakSender<T> { + chan: Arc<chan::Chan<T, Semaphore>>, +} + /// Permits to send one value into the channel. /// /// `Permit` values are returned by [`Sender::reserve()`] and [`Sender::try_reserve()`] @@ -105,9 +140,13 @@ pub struct Receiver<T> { /// } /// } /// ``` +#[track_caller] pub fn channel<T>(buffer: usize) -> (Sender<T>, Receiver<T>) { assert!(buffer > 0, "mpsc bounded channel requires buffer > 0"); - let semaphore = (semaphore::Semaphore::new(buffer), buffer); + let semaphore = Semaphore { + semaphore: semaphore::Semaphore::new(buffer), + bound: buffer, + }; let (tx, rx) = chan::channel(semaphore); let tx = Sender::new(tx); @@ -118,7 +157,11 @@ pub fn channel<T>(buffer: usize) -> (Sender<T>, Receiver<T>) { /// Channel semaphore is a tuple of the semaphore implementation and a `usize` /// representing the channel bound. -type Semaphore = (semaphore::Semaphore, usize); +#[derive(Debug)] +pub(crate) struct Semaphore { + pub(crate) semaphore: semaphore::Semaphore, + pub(crate) bound: usize, +} impl<T> Receiver<T> { pub(crate) fn new(chan: chan::Rx<T, Semaphore>) -> Receiver<T> { @@ -281,6 +324,7 @@ impl<T> Receiver<T> { /// sync_code.join().unwrap() /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] pub fn blocking_recv(&mut self) -> Option<T> { crate::future::block_on(self.recv()) @@ -535,7 +579,7 @@ impl<T> Sender<T> { /// } /// ``` pub fn try_send(&self, message: T) -> Result<(), TrySendError<T>> { - match self.chan.semaphore().0.try_acquire(1) { + match self.chan.semaphore().semaphore.try_acquire(1) { Ok(_) => {} Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(message)), Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(message)), @@ -563,6 +607,11 @@ impl<T> Sender<T> { /// [`close`]: Receiver::close /// [`Receiver`]: Receiver /// + /// # Panics + /// + /// This function panics if it is called outside the context of a Tokio + /// runtime [with time enabled](crate::runtime::Builder::enable_time). + /// /// # Examples /// /// In the following example, each call to `send_timeout` will block until the @@ -645,6 +694,7 @@ impl<T> Sender<T> { /// sync_code.join().unwrap() /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] pub fn blocking_send(&self, value: T) -> Result<(), SendError<T>> { crate::future::block_on(self.send(value)) @@ -809,7 +859,7 @@ impl<T> Sender<T> { } async fn reserve_inner(&self) -> Result<(), SendError<()>> { - match self.chan.semaphore().0.acquire(1).await { + match self.chan.semaphore().semaphore.acquire(1).await { Ok(_) => Ok(()), Err(_) => Err(SendError(())), } @@ -859,7 +909,7 @@ impl<T> Sender<T> { /// } /// ``` pub fn try_reserve(&self) -> Result<Permit<'_, T>, TrySendError<()>> { - match self.chan.semaphore().0.try_acquire(1) { + match self.chan.semaphore().semaphore.try_acquire(1) { Ok(_) => {} Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(())), Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(())), @@ -924,7 +974,7 @@ impl<T> Sender<T> { /// } /// ``` pub fn try_reserve_owned(self) -> Result<OwnedPermit<T>, TrySendError<Self>> { - match self.chan.semaphore().0.try_acquire(1) { + match self.chan.semaphore().semaphore.try_acquire(1) { Ok(_) => {} Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(self)), Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(self)), @@ -955,6 +1005,8 @@ impl<T> Sender<T> { /// /// The capacity goes down when sending a value by calling [`send`] or by reserving capacity /// with [`reserve`]. The capacity goes up when values are received by the [`Receiver`]. + /// This is distinct from [`max_capacity`], which always returns buffer capacity initially + /// specified when calling [`channel`] /// /// # Examples /// @@ -980,8 +1032,56 @@ impl<T> Sender<T> { /// /// [`send`]: Sender::send /// [`reserve`]: Sender::reserve + /// [`channel`]: channel + /// [`max_capacity`]: Sender::max_capacity pub fn capacity(&self) -> usize { - self.chan.semaphore().0.available_permits() + self.chan.semaphore().semaphore.available_permits() + } + + /// Converts the `Sender` to a [`WeakSender`] that does not count + /// towards RAII semantics, i.e. if all `Sender` instances of the + /// channel were dropped and only `WeakSender` instances remain, + /// the channel is closed. + pub fn downgrade(&self) -> WeakSender<T> { + WeakSender { + chan: self.chan.downgrade(), + } + } + + /// Returns the maximum buffer capacity of the channel. + /// + /// The maximum capacity is the buffer capacity initially specified when calling + /// [`channel`]. This is distinct from [`capacity`], which returns the *current* + /// available buffer capacity: as messages are sent and received, the + /// value returned by [`capacity`] will go up or down, whereas the value + /// returned by `max_capacity` will remain constant. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::mpsc; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, _rx) = mpsc::channel::<()>(5); + /// + /// // both max capacity and capacity are the same at first + /// assert_eq!(tx.max_capacity(), 5); + /// assert_eq!(tx.capacity(), 5); + /// + /// // Making a reservation doesn't change the max capacity. + /// let permit = tx.reserve().await.unwrap(); + /// assert_eq!(tx.max_capacity(), 5); + /// // but drops the capacity by one + /// assert_eq!(tx.capacity(), 4); + /// } + /// ``` + /// + /// [`channel`]: channel + /// [`max_capacity`]: Sender::max_capacity + /// [`capacity`]: Sender::capacity + pub fn max_capacity(&self) -> usize { + self.chan.semaphore().bound } } @@ -1001,6 +1101,29 @@ impl<T> fmt::Debug for Sender<T> { } } +impl<T> Clone for WeakSender<T> { + fn clone(&self) -> Self { + WeakSender { + chan: self.chan.clone(), + } + } +} + +impl<T> WeakSender<T> { + /// Tries to convert a WeakSender into a [`Sender`]. This will return `Some` + /// if there are other `Sender` instances alive and the channel wasn't + /// previously dropped, otherwise `None` is returned. + pub fn upgrade(&self) -> Option<Sender<T>> { + chan::Tx::upgrade(self.chan.clone()).map(Sender::new) + } +} + +impl<T> fmt::Debug for WeakSender<T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("WeakSender").finish() + } +} + // ===== impl Permit ===== impl<T> Permit<'_, T> { diff --git a/src/sync/mpsc/chan.rs b/src/sync/mpsc/chan.rs index c3007de..edd3e95 100644 --- a/src/sync/mpsc/chan.rs +++ b/src/sync/mpsc/chan.rs @@ -2,15 +2,14 @@ use crate::loom::cell::UnsafeCell; use crate::loom::future::AtomicWaker; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::Arc; -use crate::park::thread::CachedParkThread; -use crate::park::Park; +use crate::runtime::park::CachedParkThread; use crate::sync::mpsc::error::TryRecvError; -use crate::sync::mpsc::list; +use crate::sync::mpsc::{bounded, list, unbounded}; use crate::sync::notify::Notify; use std::fmt; use std::process; -use std::sync::atomic::Ordering::{AcqRel, Relaxed}; +use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use std::task::Poll::{Pending, Ready}; use std::task::{Context, Poll}; @@ -46,7 +45,7 @@ pub(crate) trait Semaphore { fn is_closed(&self) -> bool; } -struct Chan<T, S> { +pub(super) struct Chan<T, S> { /// Notifies all tasks listening for the receiver being dropped. notify_rx_closed: Notify, @@ -129,6 +128,30 @@ impl<T, S> Tx<T, S> { Tx { inner: chan } } + pub(super) fn downgrade(&self) -> Arc<Chan<T, S>> { + self.inner.clone() + } + + // Returns the upgraded channel or None if the upgrade failed. + pub(super) fn upgrade(chan: Arc<Chan<T, S>>) -> Option<Self> { + let mut tx_count = chan.tx_count.load(Acquire); + + loop { + if tx_count == 0 { + // channel is closed + return None; + } + + match chan + .tx_count + .compare_exchange_weak(tx_count, tx_count + 1, AcqRel, Acquire) + { + Ok(_) => return Some(Tx { inner: chan }), + Err(prev_count) => tx_count = prev_count, + } + } + } + pub(super) fn semaphore(&self) -> &S { &self.inner.semaphore } @@ -220,7 +243,7 @@ impl<T, S: Semaphore> Rx<T, S> { use super::block::Read::*; // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); self.inner.rx_fields.with_mut(|rx_fields_ptr| { let rx_fields = unsafe { &mut *rx_fields_ptr }; @@ -301,13 +324,13 @@ impl<T, S: Semaphore> Rx<T, S> { // Park the thread until the problematic send has completed. let mut park = CachedParkThread::new(); - let waker = park.unpark().into_waker(); + let waker = park.waker().unwrap(); loop { self.inner.rx_waker.register_by_ref(&waker); // It is possible that the problematic send has now completed, // so we have to check for messages again. try_recv!(); - park.park().expect("park failed"); + park.park(); } }) } @@ -345,7 +368,7 @@ impl<T, S> Drop for Chan<T, S> { fn drop(&mut self) { use super::block::Read::Value; - // Safety: the only owner of the rx fields is Chan, and eing + // Safety: the only owner of the rx fields is Chan, and being // inside its own Drop means we're the last ones to touch it. self.rx_fields.with_mut(|rx_fields_ptr| { let rx_fields = unsafe { &mut *rx_fields_ptr }; @@ -358,32 +381,29 @@ impl<T, S> Drop for Chan<T, S> { // ===== impl Semaphore for (::Semaphore, capacity) ===== -impl Semaphore for (crate::sync::batch_semaphore::Semaphore, usize) { +impl Semaphore for bounded::Semaphore { fn add_permit(&self) { - self.0.release(1) + self.semaphore.release(1) } fn is_idle(&self) -> bool { - self.0.available_permits() == self.1 + self.semaphore.available_permits() == self.bound } fn close(&self) { - self.0.close(); + self.semaphore.close(); } fn is_closed(&self) -> bool { - self.0.is_closed() + self.semaphore.is_closed() } } // ===== impl Semaphore for AtomicUsize ===== -use std::sync::atomic::Ordering::{Acquire, Release}; -use std::usize; - -impl Semaphore for AtomicUsize { +impl Semaphore for unbounded::Semaphore { fn add_permit(&self) { - let prev = self.fetch_sub(2, Release); + let prev = self.0.fetch_sub(2, Release); if prev >> 1 == 0 { // Something went wrong @@ -392,14 +412,14 @@ impl Semaphore for AtomicUsize { } fn is_idle(&self) -> bool { - self.load(Acquire) >> 1 == 0 + self.0.load(Acquire) >> 1 == 0 } fn close(&self) { - self.fetch_or(1, Release); + self.0.fetch_or(1, Release); } fn is_closed(&self) -> bool { - self.load(Acquire) & 1 == 1 + self.0.load(Acquire) & 1 == 1 } } diff --git a/src/sync/mpsc/error.rs b/src/sync/mpsc/error.rs index b7b9cf7..1c789da 100644 --- a/src/sync/mpsc/error.rs +++ b/src/sync/mpsc/error.rs @@ -19,7 +19,7 @@ impl<T: fmt::Debug> std::error::Error for SendError<T> {} /// This enumeration is the list of the possible error outcomes for the /// [try_send](super::Sender::try_send) method. -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq)] pub enum TrySendError<T> { /// The data could not be sent on the channel because the channel is /// currently full and sending would require blocking. @@ -78,7 +78,7 @@ impl Error for TryRecvError {} // ===== RecvError ===== /// Error returned by `Receiver`. -#[derive(Debug)] +#[derive(Debug, Clone)] #[doc(hidden)] #[deprecated(note = "This type is unused because recv returns an Option.")] pub struct RecvError(()); @@ -96,7 +96,7 @@ impl Error for RecvError {} cfg_time! { // ===== SendTimeoutError ===== - #[derive(Debug)] + #[derive(Debug, Eq, PartialEq)] /// Error returned by [`Sender::send_timeout`](super::Sender::send_timeout)]. pub enum SendTimeoutError<T> { /// The data could not be sent on the channel because the channel is diff --git a/src/sync/mpsc/list.rs b/src/sync/mpsc/list.rs index e4eeb45..10b2957 100644 --- a/src/sync/mpsc/list.rs +++ b/src/sync/mpsc/list.rs @@ -44,7 +44,7 @@ pub(crate) enum TryPopResult<T> { pub(crate) fn channel<T>() -> (Tx<T>, Rx<T>) { // Create the initial block shared between the tx and rx halves. - let initial_block = Box::new(Block::new(0)); + let initial_block = Block::new(0); let initial_block_ptr = Box::into_raw(initial_block); let tx = Tx { diff --git a/src/sync/mpsc/mod.rs b/src/sync/mpsc/mod.rs index 879e3dc..33889fa 100644 --- a/src/sync/mpsc/mod.rs +++ b/src/sync/mpsc/mod.rs @@ -21,6 +21,9 @@ //! when additional capacity is available. In other words, the channel provides //! backpressure. //! +//! This channel is also suitable for the single-producer single-consumer +//! use-case. (Unless you only need to send one message, in which case you +//! should use the [oneshot] channel.) //! //! # Disconnection //! @@ -58,6 +61,22 @@ //! [crossbeam][crossbeam-unbounded]. Similarly, for sending a message _from sync //! to async_, you should use an unbounded Tokio `mpsc` channel. //! +//! Please be aware that the above remarks were written with the `mpsc` channel +//! in mind, but they can also be generalized to other kinds of channels. In +//! general, any channel method that isn't marked async can be called anywhere, +//! including outside of the runtime. For example, sending a message on a +//! [oneshot] channel from outside the runtime is perfectly fine. +//! +//! # Multiple runtimes +//! +//! The mpsc channel does not care about which runtime you use it in, and can be +//! used to send messages from one runtime to another. It can also be used in +//! non-Tokio runtimes. +//! +//! There is one exception to the above: the [`send_timeout`] must be used from +//! within a Tokio runtime, however it is still not tied to one specific Tokio +//! runtime, and the sender may be moved from one Tokio runtime to another. +//! //! [`Sender`]: crate::sync::mpsc::Sender //! [`Receiver`]: crate::sync::mpsc::Receiver //! [bounded-send]: crate::sync::mpsc::Sender::send() @@ -66,21 +85,25 @@ //! [blocking-recv]: crate::sync::mpsc::Receiver::blocking_recv() //! [`UnboundedSender`]: crate::sync::mpsc::UnboundedSender //! [`UnboundedReceiver`]: crate::sync::mpsc::UnboundedReceiver +//! [oneshot]: crate::sync::oneshot //! [`Handle::block_on`]: crate::runtime::Handle::block_on() //! [std-unbounded]: std::sync::mpsc::channel //! [crossbeam-unbounded]: https://docs.rs/crossbeam/*/crossbeam/channel/fn.unbounded.html +//! [`send_timeout`]: crate::sync::mpsc::Sender::send_timeout pub(super) mod block; mod bounded; -pub use self::bounded::{channel, OwnedPermit, Permit, Receiver, Sender}; +pub use self::bounded::{channel, OwnedPermit, Permit, Receiver, Sender, WeakSender}; mod chan; pub(super) mod list; mod unbounded; -pub use self::unbounded::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +pub use self::unbounded::{ + unbounded_channel, UnboundedReceiver, UnboundedSender, WeakUnboundedSender, +}; pub mod error; diff --git a/src/sync/mpsc/unbounded.rs b/src/sync/mpsc/unbounded.rs index b133f9f..5010204 100644 --- a/src/sync/mpsc/unbounded.rs +++ b/src/sync/mpsc/unbounded.rs @@ -1,4 +1,4 @@ -use crate::loom::sync::atomic::AtomicUsize; +use crate::loom::sync::{atomic::AtomicUsize, Arc}; use crate::sync::mpsc::chan; use crate::sync::mpsc::error::{SendError, TryRecvError}; @@ -13,6 +13,40 @@ pub struct UnboundedSender<T> { chan: chan::Tx<T, Semaphore>, } +/// An unbounded sender that does not prevent the channel from being closed. +/// +/// If all [`UnboundedSender`] instances of a channel were dropped and only +/// `WeakUnboundedSender` instances remain, the channel is closed. +/// +/// In order to send messages, the `WeakUnboundedSender` needs to be upgraded using +/// [`WeakUnboundedSender::upgrade`], which returns `Option<UnboundedSender>`. It returns `None` +/// if all `UnboundedSender`s have been dropped, and otherwise it returns an `UnboundedSender`. +/// +/// [`UnboundedSender`]: UnboundedSender +/// [`WeakUnboundedSender::upgrade`]: WeakUnboundedSender::upgrade +/// +/// #Examples +/// +/// ``` +/// use tokio::sync::mpsc::unbounded_channel; +/// +/// #[tokio::main] +/// async fn main() { +/// let (tx, _rx) = unbounded_channel::<i32>(); +/// let tx_weak = tx.downgrade(); +/// +/// // Upgrading will succeed because `tx` still exists. +/// assert!(tx_weak.upgrade().is_some()); +/// +/// // If we drop `tx`, then it will fail. +/// drop(tx); +/// assert!(tx_weak.clone().upgrade().is_none()); +/// } +/// ``` +pub struct WeakUnboundedSender<T> { + chan: Arc<chan::Chan<T, Semaphore>>, +} + impl<T> Clone for UnboundedSender<T> { fn clone(&self) -> Self { UnboundedSender { @@ -61,7 +95,7 @@ impl<T> fmt::Debug for UnboundedReceiver<T> { /// the channel. Using an `unbounded` channel has the ability of causing the /// process to run out of memory. In this case, the process will be aborted. pub fn unbounded_channel<T>() -> (UnboundedSender<T>, UnboundedReceiver<T>) { - let (tx, rx) = chan::channel(AtomicUsize::new(0)); + let (tx, rx) = chan::channel(Semaphore(AtomicUsize::new(0))); let tx = UnboundedSender::new(tx); let rx = UnboundedReceiver::new(rx); @@ -70,7 +104,8 @@ pub fn unbounded_channel<T>() -> (UnboundedSender<T>, UnboundedReceiver<T>) { } /// No capacity -type Semaphore = AtomicUsize; +#[derive(Debug)] +pub(crate) struct Semaphore(pub(crate) AtomicUsize); impl<T> UnboundedReceiver<T> { pub(crate) fn new(chan: chan::Rx<T, Semaphore>) -> UnboundedReceiver<T> { @@ -79,8 +114,14 @@ impl<T> UnboundedReceiver<T> { /// Receives the next value for this receiver. /// - /// `None` is returned when all `Sender` halves have dropped, indicating - /// that no further values can be sent on the channel. + /// This method returns `None` if the channel has been closed and there are + /// no remaining messages in the channel's buffer. This indicates that no + /// further values can ever be received from this `Receiver`. The channel is + /// closed when all senders have been dropped, or when [`close`] is called. + /// + /// If there are no messages in the channel's buffer, but the channel has + /// not yet been closed, this method will sleep until a message is sent or + /// the channel is closed. /// /// # Cancel safety /// @@ -89,6 +130,8 @@ impl<T> UnboundedReceiver<T> { /// completes first, it is guaranteed that no messages were received on this /// channel. /// + /// [`close`]: Self::close + /// /// # Examples /// /// ``` @@ -198,6 +241,7 @@ impl<T> UnboundedReceiver<T> { /// sync_code.join().unwrap(); /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] pub fn blocking_recv(&mut self) -> Option<T> { crate::future::block_on(self.recv()) @@ -207,6 +251,9 @@ impl<T> UnboundedReceiver<T> { /// /// This prevents any further messages from being sent on the channel while /// still enabling the receiver to drain messages that are buffered. + /// + /// To guarantee that no messages are dropped, after calling `close()`, + /// `recv()` must be called until `None` is returned. pub fn close(&mut self) { self.chan.close(); } @@ -267,7 +314,7 @@ impl<T> UnboundedSender<T> { use std::process; use std::sync::atomic::Ordering::{AcqRel, Acquire}; - let mut curr = self.chan.semaphore().load(Acquire); + let mut curr = self.chan.semaphore().0.load(Acquire); loop { if curr & 1 == 1 { @@ -283,6 +330,7 @@ impl<T> UnboundedSender<T> { match self .chan .semaphore() + .0 .compare_exchange(curr, curr + 2, AcqRel, Acquire) { Ok(_) => return true, @@ -370,4 +418,37 @@ impl<T> UnboundedSender<T> { pub fn same_channel(&self, other: &Self) -> bool { self.chan.same_channel(&other.chan) } + + /// Converts the `UnboundedSender` to a [`WeakUnboundedSender`] that does not count + /// towards RAII semantics, i.e. if all `UnboundedSender` instances of the + /// channel were dropped and only `WeakUnboundedSender` instances remain, + /// the channel is closed. + pub fn downgrade(&self) -> WeakUnboundedSender<T> { + WeakUnboundedSender { + chan: self.chan.downgrade(), + } + } +} + +impl<T> Clone for WeakUnboundedSender<T> { + fn clone(&self) -> Self { + WeakUnboundedSender { + chan: self.chan.clone(), + } + } +} + +impl<T> WeakUnboundedSender<T> { + /// Tries to convert a WeakUnboundedSender into an [`UnboundedSender`]. + /// This will return `Some` if there are other `Sender` instances alive and + /// the channel wasn't previously dropped, otherwise `None` is returned. + pub fn upgrade(&self) -> Option<UnboundedSender<T>> { + chan::Tx::upgrade(self.chan.clone()).map(UnboundedSender::new) + } +} + +impl<T> fmt::Debug for WeakUnboundedSender<T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("WeakUnboundedSender").finish() + } } diff --git a/src/sync/mutex.rs b/src/sync/mutex.rs index 4d9f988..024755c 100644 --- a/src/sync/mutex.rs +++ b/src/sync/mutex.rs @@ -1,6 +1,8 @@ #![cfg_attr(not(feature = "sync"), allow(unreachable_pub, dead_code))] use crate::sync::batch_semaphore as semaphore; +#[cfg(all(tokio_unstable, feature = "tracing"))] +use crate::util::trace; use std::cell::UnsafeCell; use std::error::Error; @@ -124,6 +126,8 @@ use std::{fmt, marker, mem}; /// [`Send`]: trait@std::marker::Send /// [`lock`]: method@Mutex::lock pub struct Mutex<T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, s: semaphore::Semaphore, c: UnsafeCell<T>, } @@ -137,7 +141,10 @@ pub struct Mutex<T: ?Sized> { /// /// The lock is automatically released whenever the guard is dropped, at which /// point `lock` will succeed yet again. +#[must_use = "if unused the Mutex will immediately unlock"] pub struct MutexGuard<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, lock: &'a Mutex<T>, } @@ -157,6 +164,8 @@ pub struct MutexGuard<'a, T: ?Sized> { /// /// [`Arc`]: std::sync::Arc pub struct OwnedMutexGuard<T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, lock: Arc<Mutex<T>>, } @@ -191,8 +200,8 @@ unsafe impl<'a, T> Send for MappedMutexGuard<'a, T> where T: ?Sized + Send + 'a /// `RwLock::try_read` operation will only fail if the lock is currently held /// by an exclusive writer. /// -/// `RwLock::try_write` operation will if lock is held by any reader or by an -/// exclusive writer. +/// `RwLock::try_write` operation will only fail if the lock is currently held +/// by any reader or by an exclusive writer. /// /// [`Mutex::try_lock`]: Mutex::try_lock /// [`RwLock::try_read`]: fn@super::RwLock::try_read @@ -242,13 +251,42 @@ impl<T: ?Sized> Mutex<T> { /// /// let lock = Mutex::new(5); /// ``` + #[track_caller] pub fn new(t: T) -> Self where T: Sized, { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let location = std::panic::Location::caller(); + + tracing::trace_span!( + "runtime.resource", + concrete_type = "Mutex", + kind = "Sync", + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + ) + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let s = resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = false, + ); + semaphore::Semaphore::new(1) + }); + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + let s = semaphore::Semaphore::new(1); + Self { c: UnsafeCell::new(t), - s: semaphore::Semaphore::new(1), + s, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -270,6 +308,8 @@ impl<T: ?Sized> Mutex<T> { Self { c: UnsafeCell::new(t), s: semaphore::Semaphore::const_new(1), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span::none(), } } @@ -297,16 +337,50 @@ impl<T: ?Sized> Mutex<T> { /// } /// ``` pub async fn lock(&self) -> MutexGuard<'_, T> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + trace::async_op( + || self.acquire(), + self.resource_span.clone(), + "Mutex::lock", + "poll", + false, + ) + .await; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = true, + ); + }); + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] self.acquire().await; - MutexGuard { lock: self } + + MutexGuard { + lock: self, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + } } - /// Blocking lock this mutex. When the lock has been acquired, function returns a + /// Blockingly locks this `Mutex`. When the lock has been acquired, function returns a /// [`MutexGuard`]. /// /// This method is intended for use cases where you /// need to use this mutex in asynchronous code as well as in synchronous code. /// + /// # Panics + /// + /// This function panics if called within an asynchronous execution context. + /// + /// - If you find yourself in an asynchronous execution context and needing + /// to call some (synchronous) function which performs one of these + /// `blocking_` operations, then consider wrapping that call inside + /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking] + /// (or [`block_in_place()`][crate::task::block_in_place]). + /// /// # Examples /// /// ``` @@ -316,25 +390,90 @@ impl<T: ?Sized> Mutex<T> { /// #[tokio::main] /// async fn main() { /// let mutex = Arc::new(Mutex::new(1)); + /// let lock = mutex.lock().await; /// /// let mutex1 = Arc::clone(&mutex); - /// let sync_code = tokio::task::spawn_blocking(move || { + /// let blocking_task = tokio::task::spawn_blocking(move || { + /// // This shall block until the `lock` is released. /// let mut n = mutex1.blocking_lock(); /// *n = 2; /// }); /// - /// sync_code.await.unwrap(); + /// assert_eq!(*lock, 1); + /// // Release the lock. + /// drop(lock); /// - /// let n = mutex.lock().await; + /// // Await the completion of the blocking task. + /// blocking_task.await.unwrap(); + /// + /// // Assert uncontended. + /// let n = mutex.try_lock().unwrap(); /// assert_eq!(*n, 2); /// } /// /// ``` + #[track_caller] #[cfg(feature = "sync")] pub fn blocking_lock(&self) -> MutexGuard<'_, T> { crate::future::block_on(self.lock()) } + /// Blockingly locks this `Mutex`. When the lock has been acquired, function returns an + /// [`OwnedMutexGuard`]. + /// + /// This method is identical to [`Mutex::blocking_lock`], except that the returned + /// guard references the `Mutex` with an [`Arc`] rather than by borrowing + /// it. Therefore, the `Mutex` must be wrapped in an `Arc` to call this + /// method, and the guard will live for the `'static` lifetime, as it keeps + /// the `Mutex` alive by holding an `Arc`. + /// + /// # Panics + /// + /// This function panics if called within an asynchronous execution context. + /// + /// - If you find yourself in an asynchronous execution context and needing + /// to call some (synchronous) function which performs one of these + /// `blocking_` operations, then consider wrapping that call inside + /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking] + /// (or [`block_in_place()`][crate::task::block_in_place]). + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// use tokio::sync::Mutex; + /// + /// #[tokio::main] + /// async fn main() { + /// let mutex = Arc::new(Mutex::new(1)); + /// let lock = mutex.lock().await; + /// + /// let mutex1 = Arc::clone(&mutex); + /// let blocking_task = tokio::task::spawn_blocking(move || { + /// // This shall block until the `lock` is released. + /// let mut n = mutex1.blocking_lock_owned(); + /// *n = 2; + /// }); + /// + /// assert_eq!(*lock, 1); + /// // Release the lock. + /// drop(lock); + /// + /// // Await the completion of the blocking task. + /// blocking_task.await.unwrap(); + /// + /// // Assert uncontended. + /// let n = mutex.try_lock().unwrap(); + /// assert_eq!(*n, 2); + /// } + /// + /// ``` + #[track_caller] + #[cfg(feature = "sync")] + pub fn blocking_lock_owned(self: Arc<Self>) -> OwnedMutexGuard<T> { + crate::future::block_on(self.lock_owned()) + } + /// Locks this mutex, causing the current task to yield until the lock has /// been acquired. When the lock has been acquired, this returns an /// [`OwnedMutexGuard`]. @@ -368,8 +507,35 @@ impl<T: ?Sized> Mutex<T> { /// /// [`Arc`]: std::sync::Arc pub async fn lock_owned(self: Arc<Self>) -> OwnedMutexGuard<T> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + trace::async_op( + || self.acquire(), + self.resource_span.clone(), + "Mutex::lock_owned", + "poll", + false, + ) + .await; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = true, + ); + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] self.acquire().await; - OwnedMutexGuard { lock: self } + + OwnedMutexGuard { + lock: self, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, + } } async fn acquire(&self) { @@ -399,7 +565,21 @@ impl<T: ?Sized> Mutex<T> { /// ``` pub fn try_lock(&self) -> Result<MutexGuard<'_, T>, TryLockError> { match self.s.try_acquire(1) { - Ok(_) => Ok(MutexGuard { lock: self }), + Ok(_) => { + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = true, + ); + }); + + Ok(MutexGuard { + lock: self, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + }) + } Err(_) => Err(TryLockError(())), } } @@ -454,7 +634,24 @@ impl<T: ?Sized> Mutex<T> { /// # } pub fn try_lock_owned(self: Arc<Self>) -> Result<OwnedMutexGuard<T>, TryLockError> { match self.s.try_acquire(1) { - Ok(_) => Ok(OwnedMutexGuard { lock: self }), + Ok(_) => { + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = true, + ); + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + + Ok(OwnedMutexGuard { + lock: self, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, + }) + } Err(_) => Err(TryLockError(())), } } @@ -626,7 +823,7 @@ impl<'a, T: ?Sized> MutexGuard<'a, T> { /// # async fn main() { /// # let mutex = Mutex::new(0u32); /// # let guard = mutex.lock().await; - /// # unlock_and_relock(guard).await; + /// # let _guard = unlock_and_relock(guard).await; /// # } /// ``` #[inline] @@ -637,7 +834,14 @@ impl<'a, T: ?Sized> MutexGuard<'a, T> { impl<T: ?Sized> Drop for MutexGuard<'_, T> { fn drop(&mut self) { - self.lock.s.release(1) + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = false, + ); + }); + self.lock.s.release(1); } } @@ -699,6 +903,13 @@ impl<T: ?Sized> OwnedMutexGuard<T> { impl<T: ?Sized> Drop for OwnedMutexGuard<T> { fn drop(&mut self) { + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = false, + ); + }); self.lock.s.release(1) } } diff --git a/src/sync/notify.rs b/src/sync/notify.rs index c93ce3b..efe16f9 100644 --- a/src/sync/notify.rs +++ b/src/sync/notify.rs @@ -13,6 +13,7 @@ use crate::util::WakeList; use std::cell::UnsafeCell; use std::future::Future; use std::marker::PhantomPinned; +use std::panic::{RefUnwindSafe, UnwindSafe}; use std::pin::Pin; use std::ptr::NonNull; use std::sync::atomic::Ordering::SeqCst; @@ -26,22 +27,23 @@ type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>; /// `Notify` itself does not carry any data. Instead, it is to be used to signal /// another task to perform an operation. /// -/// `Notify` can be thought of as a [`Semaphore`] starting with 0 permits. -/// [`notified().await`] waits for a permit to become available, and [`notify_one()`] -/// sets a permit **if there currently are no available permits**. +/// A `Notify` can be thought of as a [`Semaphore`] starting with 0 permits. The +/// [`notified().await`] method waits for a permit to become available, and +/// [`notify_one()`] sets a permit **if there currently are no available +/// permits**. /// /// The synchronization details of `Notify` are similar to /// [`thread::park`][park] and [`Thread::unpark`][unpark] from std. A [`Notify`] /// value contains a single permit. [`notified().await`] waits for the permit to -/// be made available, consumes the permit, and resumes. [`notify_one()`] sets the -/// permit, waking a pending task if there is one. +/// be made available, consumes the permit, and resumes. [`notify_one()`] sets +/// the permit, waking a pending task if there is one. /// -/// If `notify_one()` is called **before** `notified().await`, then the next call to -/// `notified().await` will complete immediately, consuming the permit. Any -/// subsequent calls to `notified().await` will wait for a new permit. +/// If `notify_one()` is called **before** `notified().await`, then the next +/// call to `notified().await` will complete immediately, consuming the permit. +/// Any subsequent calls to `notified().await` will wait for a new permit. /// -/// If `notify_one()` is called **multiple** times before `notified().await`, only a -/// **single** permit is stored. The next call to `notified().await` will +/// If `notify_one()` is called **multiple** times before `notified().await`, +/// only a **single** permit is stored. The next call to `notified().await` will /// complete immediately, but the one after will wait for a new permit. /// /// # Examples @@ -70,7 +72,11 @@ type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>; /// } /// ``` /// -/// Unbound mpsc channel. +/// Unbound multi-producer single-consumer (mpsc) channel. +/// +/// No wakeups can be lost when using this channel because the call to +/// `notify_one()` will store a permit in the `Notify`, which the following call +/// to `notified()` will consume. /// /// ``` /// use tokio::sync::Notify; @@ -92,6 +98,8 @@ type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>; /// self.notify.notify_one(); /// } /// +/// // This is a single-consumer channel, so several concurrent calls to +/// // `recv` are not allowed. /// pub async fn recv(&self) -> T { /// loop { /// // Drain values @@ -106,10 +114,87 @@ type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>; /// } /// ``` /// +/// Unbound multi-producer multi-consumer (mpmc) channel. +/// +/// The call to [`enable`] is important because otherwise if you have two +/// calls to `recv` and two calls to `send` in parallel, the following could +/// happen: +/// +/// 1. Both calls to `try_recv` return `None`. +/// 2. Both new elements are added to the vector. +/// 3. The `notify_one` method is called twice, adding only a single +/// permit to the `Notify`. +/// 4. Both calls to `recv` reach the `Notified` future. One of them +/// consumes the permit, and the other sleeps forever. +/// +/// By adding the `Notified` futures to the list by calling `enable` before +/// `try_recv`, the `notify_one` calls in step three would remove the +/// futures from the list and mark them notified instead of adding a permit +/// to the `Notify`. This ensures that both futures are woken. +/// +/// Notice that this failure can only happen if there are two concurrent calls +/// to `recv`. This is why the mpsc example above does not require a call to +/// `enable`. +/// +/// ``` +/// use tokio::sync::Notify; +/// +/// use std::collections::VecDeque; +/// use std::sync::Mutex; +/// +/// struct Channel<T> { +/// messages: Mutex<VecDeque<T>>, +/// notify_on_sent: Notify, +/// } +/// +/// impl<T> Channel<T> { +/// pub fn send(&self, msg: T) { +/// let mut locked_queue = self.messages.lock().unwrap(); +/// locked_queue.push_back(msg); +/// drop(locked_queue); +/// +/// // Send a notification to one of the calls currently +/// // waiting in a call to `recv`. +/// self.notify_on_sent.notify_one(); +/// } +/// +/// pub fn try_recv(&self) -> Option<T> { +/// let mut locked_queue = self.messages.lock().unwrap(); +/// locked_queue.pop_front() +/// } +/// +/// pub async fn recv(&self) -> T { +/// let future = self.notify_on_sent.notified(); +/// tokio::pin!(future); +/// +/// loop { +/// // Make sure that no wakeup is lost if we get +/// // `None` from `try_recv`. +/// future.as_mut().enable(); +/// +/// if let Some(msg) = self.try_recv() { +/// return msg; +/// } +/// +/// // Wait for a call to `notify_one`. +/// // +/// // This uses `.as_mut()` to avoid consuming the future, +/// // which lets us call `Pin::set` below. +/// future.as_mut().await; +/// +/// // Reset the future in case another call to +/// // `try_recv` got the message before us. +/// future.set(self.notify_on_sent.notified()); +/// } +/// } +/// } +/// ``` +/// /// [park]: std::thread::park /// [unpark]: std::thread::Thread::unpark /// [`notified().await`]: Notify::notified() /// [`notify_one()`]: Notify::notify_one() +/// [`enable`]: Notified::enable() /// [`Semaphore`]: crate::sync::Semaphore #[derive(Debug)] pub struct Notify { @@ -144,7 +229,18 @@ struct Waiter { _p: PhantomPinned, } -/// Future returned from [`Notify::notified()`] +generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> { + &self.pointers + } + } +} + +/// Future returned from [`Notify::notified()`]. +/// +/// This future is fused, so once it has completed, any future calls to poll +/// will immediately return `Poll::Ready`. #[derive(Debug)] pub struct Notified<'a> { /// The `Notify` being received on. @@ -248,7 +344,16 @@ impl Notify { /// immediately, consuming that permit. Otherwise, `notified().await` waits /// for a permit to be made available by the next call to `notify_one()`. /// + /// The `Notified` future is not guaranteed to receive wakeups from calls to + /// `notify_one()` if it has not yet been polled. See the documentation for + /// [`Notified::enable()`] for more details. + /// + /// The `Notified` future is guaranteed to receive wakeups from + /// `notify_waiters()` as soon as it has been created, even if it has not + /// yet been polled. + /// /// [`notify_one()`]: Notify::notify_one + /// [`Notified::enable()`]: Notified::enable /// /// # Cancel safety /// @@ -404,7 +509,7 @@ impl Notify { // transition out of WAITING while the lock is held. let curr = self.state.load(SeqCst); - if let EMPTY | NOTIFIED = get_state(curr) { + if matches!(get_state(curr), EMPTY | NOTIFIED) { // There are no waiting tasks. All we need to do is increment the // number of times this method was called. atomic_inc_num_notify_waiters_calls(&self.state); @@ -462,6 +567,9 @@ impl Default for Notify { } } +impl UnwindSafe for Notify {} +impl RefUnwindSafe for Notify {} + fn notify_locked(waiters: &mut WaitList, state: &AtomicUsize, curr: usize) -> Option<Waker> { loop { match get_state(curr) { @@ -512,6 +620,114 @@ fn notify_locked(waiters: &mut WaitList, state: &AtomicUsize, curr: usize) -> Op // ===== impl Notified ===== impl Notified<'_> { + /// Adds this future to the list of futures that are ready to receive + /// wakeups from calls to [`notify_one`]. + /// + /// Polling the future also adds it to the list, so this method should only + /// be used if you want to add the future to the list before the first call + /// to `poll`. (In fact, this method is equivalent to calling `poll` except + /// that no `Waker` is registered.) + /// + /// This has no effect on notifications sent using [`notify_waiters`], which + /// are received as long as they happen after the creation of the `Notified` + /// regardless of whether `enable` or `poll` has been called. + /// + /// This method returns true if the `Notified` is ready. This happens in the + /// following situations: + /// + /// 1. The `notify_waiters` method was called between the creation of the + /// `Notified` and the call to this method. + /// 2. This is the first call to `enable` or `poll` on this future, and the + /// `Notify` was holding a permit from a previous call to `notify_one`. + /// The call consumes the permit in that case. + /// 3. The future has previously been enabled or polled, and it has since + /// then been marked ready by either consuming a permit from the + /// `Notify`, or by a call to `notify_one` or `notify_waiters` that + /// removed it from the list of futures ready to receive wakeups. + /// + /// If this method returns true, any future calls to poll on the same future + /// will immediately return `Poll::Ready`. + /// + /// # Examples + /// + /// Unbound multi-producer multi-consumer (mpmc) channel. + /// + /// The call to `enable` is important because otherwise if you have two + /// calls to `recv` and two calls to `send` in parallel, the following could + /// happen: + /// + /// 1. Both calls to `try_recv` return `None`. + /// 2. Both new elements are added to the vector. + /// 3. The `notify_one` method is called twice, adding only a single + /// permit to the `Notify`. + /// 4. Both calls to `recv` reach the `Notified` future. One of them + /// consumes the permit, and the other sleeps forever. + /// + /// By adding the `Notified` futures to the list by calling `enable` before + /// `try_recv`, the `notify_one` calls in step three would remove the + /// futures from the list and mark them notified instead of adding a permit + /// to the `Notify`. This ensures that both futures are woken. + /// + /// ``` + /// use tokio::sync::Notify; + /// + /// use std::collections::VecDeque; + /// use std::sync::Mutex; + /// + /// struct Channel<T> { + /// messages: Mutex<VecDeque<T>>, + /// notify_on_sent: Notify, + /// } + /// + /// impl<T> Channel<T> { + /// pub fn send(&self, msg: T) { + /// let mut locked_queue = self.messages.lock().unwrap(); + /// locked_queue.push_back(msg); + /// drop(locked_queue); + /// + /// // Send a notification to one of the calls currently + /// // waiting in a call to `recv`. + /// self.notify_on_sent.notify_one(); + /// } + /// + /// pub fn try_recv(&self) -> Option<T> { + /// let mut locked_queue = self.messages.lock().unwrap(); + /// locked_queue.pop_front() + /// } + /// + /// pub async fn recv(&self) -> T { + /// let future = self.notify_on_sent.notified(); + /// tokio::pin!(future); + /// + /// loop { + /// // Make sure that no wakeup is lost if we get + /// // `None` from `try_recv`. + /// future.as_mut().enable(); + /// + /// if let Some(msg) = self.try_recv() { + /// return msg; + /// } + /// + /// // Wait for a call to `notify_one`. + /// // + /// // This uses `.as_mut()` to avoid consuming the future, + /// // which lets us call `Pin::set` below. + /// future.as_mut().await; + /// + /// // Reset the future in case another call to + /// // `try_recv` got the message before us. + /// future.set(self.notify_on_sent.notified()); + /// } + /// } + /// } + /// ``` + /// + /// [`notify_one`]: Notify::notify_one() + /// [`notify_waiters`]: Notify::notify_waiters() + pub fn enable(self: Pin<&mut Self>) -> bool { + self.poll_notified(None).is_ready() + } + /// A custom `project` implementation is used in place of `pin-project-lite` /// as a custom drop implementation is needed. fn project(self: Pin<&mut Self>) -> (&Notify, &mut State, &UnsafeCell<Waiter>) { @@ -525,12 +741,8 @@ impl Notified<'_> { (me.notify, &mut me.state, &me.waiter) } } -} -impl Future for Notified<'_> { - type Output = (); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + fn poll_notified(self: Pin<&mut Self>, waker: Option<&Waker>) -> Poll<()> { use State::*; let (notify, state, waiter) = self.project(); @@ -556,7 +768,7 @@ impl Future for Notified<'_> { // Clone the waker before locking, a waker clone can be // triggering arbitrary code. - let waker = cx.waker().clone(); + let waker = waker.cloned(); // Acquire the lock and attempt to transition to the waiting // state. @@ -617,9 +829,11 @@ impl Future for Notified<'_> { } } - // Safety: called while locked. - unsafe { - (*waiter.get()).waker = Some(waker); + if waker.is_some() { + // Safety: called while locked. + unsafe { + (*waiter.get()).waker = waker; + } } // Insert the waiter into the linked list @@ -651,8 +865,14 @@ impl Future for Notified<'_> { *state = Done; } else { // Update the waker, if necessary. - if !w.waker.as_ref().unwrap().will_wake(cx.waker()) { - w.waker = Some(cx.waker().clone()); + if let Some(waker) = waker { + let should_update = match w.waker.as_ref() { + Some(current_waker) => !current_waker.will_wake(waker), + None => true, + }; + if should_update { + w.waker = Some(waker.clone()); + } } return Poll::Pending; @@ -673,6 +893,14 @@ impl Future for Notified<'_> { } } +impl Future for Notified<'_> { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + self.poll_notified(Some(cx.waker())) + } +} + impl Drop for Notified<'_> { fn drop(&mut self) { use State::*; @@ -683,7 +911,7 @@ impl Drop for Notified<'_> { // This is where we ensure safety. The `Notified` value is being // dropped, which means we must ensure that the waiter entry is no // longer stored in the linked list. - if let Waiting = *state { + if matches!(*state, Waiting) { let mut waiters = notify.waiters.lock(); let mut notify_state = notify.state.load(SeqCst); @@ -693,11 +921,9 @@ impl Drop for Notified<'_> { // being the only `LinkedList` available to the type. unsafe { waiters.remove(NonNull::new_unchecked(waiter.get())) }; - if waiters.is_empty() { - if let WAITING = get_state(notify_state) { - notify_state = set_state(notify_state, EMPTY); - notify.state.store(notify_state, SeqCst); - } + if waiters.is_empty() && get_state(notify_state) == WAITING { + notify_state = set_state(notify_state, EMPTY); + notify.state.store(notify_state, SeqCst); } // See if the node was notified but not received. In this case, if @@ -706,7 +932,10 @@ impl Drop for Notified<'_> { // // Safety: with the entry removed from the linked list, there can be // no concurrent access to the entry - if let Some(NotificationType::OneWaiter) = unsafe { (*waiter.get()).notified } { + if matches!( + unsafe { (*waiter.get()).notified }, + Some(NotificationType::OneWaiter) + ) { if let Some(waker) = notify_locked(&mut waiters, ¬ify.state, notify_state) { drop(waiters); waker.wake(); @@ -731,8 +960,8 @@ unsafe impl linked_list::Link for Waiter { ptr } - unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> { + Waiter::addr_of_pointers(target) } } diff --git a/src/sync/once_cell.rs b/src/sync/once_cell.rs index d31a40e..081efae 100644 --- a/src/sync/once_cell.rs +++ b/src/sync/once_cell.rs @@ -106,7 +106,7 @@ impl<T> Drop for OnceCell<T> { if self.initialized_mut() { unsafe { self.value - .with_mut(|ptr| ptr::drop_in_place((&mut *ptr).as_mut_ptr())); + .with_mut(|ptr| ptr::drop_in_place((*ptr).as_mut_ptr())); }; } } @@ -416,7 +416,7 @@ unsafe impl<T: Send> Send for OnceCell<T> {} /// Errors that can be returned from [`OnceCell::set`]. /// /// [`OnceCell::set`]: crate::sync::OnceCell::set -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum SetError<T> { /// The cell was already initialized when [`OnceCell::set`] was called. /// diff --git a/src/sync/oneshot.rs b/src/sync/oneshot.rs index 4fb22ec..fcd7a32 100644 --- a/src/sync/oneshot.rs +++ b/src/sync/oneshot.rs @@ -9,6 +9,9 @@ //! //! Each handle can be used on separate tasks. //! +//! Since the `send` method is not async, it can be used anywhere. This includes +//! sending between two runtimes, and using it from non-async code. +//! //! # Examples //! //! ``` @@ -119,6 +122,8 @@ use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::Arc; +#[cfg(all(tokio_unstable, feature = "tracing"))] +use crate::util::trace; use std::fmt; use std::future::Future; @@ -212,6 +217,8 @@ use std::task::{Context, Poll, Waker}; #[derive(Debug)] pub struct Sender<T> { inner: Option<Arc<Inner<T>>>, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, } /// Receives a value from the associated [`Sender`]. @@ -220,7 +227,15 @@ pub struct Sender<T> { /// [`channel`](fn@channel) function. /// /// This channel has no `recv` method because the receiver itself implements the -/// [`Future`] trait. To receive a value, `.await` the `Receiver` object directly. +/// [`Future`] trait. To receive a `Result<T, `[`error::RecvError`]`>`, `.await` the `Receiver` object directly. +/// +/// The `poll` method on the `Future` trait is allowed to spuriously return +/// `Poll::Pending` even if the message has been sent. If such a spurious +/// failure happens, then the caller will be woken when the spurious failure has +/// been resolved so that the caller can attempt to receive the message again. +/// Note that receiving such a wakeup does not guarantee that the next call will +/// succeed — it could fail with another spurious failure. (A spurious failure +/// does not mean that the message is lost. It is just delayed.) /// /// [`Future`]: trait@std::future::Future /// @@ -302,6 +317,12 @@ pub struct Sender<T> { #[derive(Debug)] pub struct Receiver<T> { inner: Option<Arc<Inner<T>>>, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + #[cfg(all(tokio_unstable, feature = "tracing"))] + async_op_span: tracing::Span, + #[cfg(all(tokio_unstable, feature = "tracing"))] + async_op_poll_span: tracing::Span, } pub mod error { @@ -310,11 +331,13 @@ pub mod error { use std::fmt; /// Error returned by the `Future` implementation for `Receiver`. - #[derive(Debug, Eq, PartialEq)] + /// + /// This error is returned by the receiver when the sender is dropped without sending. + #[derive(Debug, Eq, PartialEq, Clone)] pub struct RecvError(pub(super) ()); /// Error returned by the `try_recv` function on `Receiver`. - #[derive(Debug, Eq, PartialEq)] + #[derive(Debug, Eq, PartialEq, Clone)] pub enum TryRecvError { /// The send half of the channel has not yet sent a value. Empty, @@ -386,21 +409,21 @@ impl Task { F: FnOnce(&Waker) -> R, { self.0.with(|ptr| { - let waker: *const Waker = (&*ptr).as_ptr(); + let waker: *const Waker = (*ptr).as_ptr(); f(&*waker) }) } unsafe fn drop_task(&self) { self.0.with_mut(|ptr| { - let ptr: *mut Waker = (&mut *ptr).as_mut_ptr(); + let ptr: *mut Waker = (*ptr).as_mut_ptr(); ptr.drop_in_place(); }); } unsafe fn set_task(&self, cx: &mut Context<'_>) { self.0.with_mut(|ptr| { - let ptr: *mut Waker = (&mut *ptr).as_mut_ptr(); + let ptr: *mut Waker = (*ptr).as_mut_ptr(); ptr.write(cx.waker().clone()); }); } @@ -439,7 +462,56 @@ struct State(usize); /// } /// } /// ``` +#[track_caller] pub fn channel<T>() -> (Sender<T>, Receiver<T>) { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let location = std::panic::Location::caller(); + + let resource_span = tracing::trace_span!( + "runtime.resource", + concrete_type = "Sender|Receiver", + kind = "Sync", + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + ); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + tx_dropped = false, + tx_dropped.op = "override", + ) + }); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + rx_dropped = false, + rx_dropped.op = "override", + ) + }); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + value_sent = false, + value_sent.op = "override", + ) + }); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + value_received = false, + value_received.op = "override", + ) + }); + + resource_span + }; + let inner = Arc::new(Inner { state: AtomicUsize::new(State::new().as_usize()), value: UnsafeCell::new(None), @@ -449,8 +521,27 @@ pub fn channel<T>() -> (Sender<T>, Receiver<T>) { let tx = Sender { inner: Some(inner.clone()), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: resource_span.clone(), + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let async_op_span = resource_span + .in_scope(|| tracing::trace_span!("runtime.resource.async_op", source = "Receiver::await")); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let async_op_poll_span = + async_op_span.in_scope(|| tracing::trace_span!("runtime.resource.async_op.poll")); + + let rx = Receiver { + inner: Some(inner), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, + #[cfg(all(tokio_unstable, feature = "tracing"))] + async_op_span, + #[cfg(all(tokio_unstable, feature = "tracing"))] + async_op_poll_span, }; - let rx = Receiver { inner: Some(inner) }; (tx, rx) } @@ -522,6 +613,15 @@ impl<T> Sender<T> { } } + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + value_sent = true, + value_sent.op = "override", + ) + }); + Ok(()) } @@ -595,7 +695,20 @@ impl<T> Sender<T> { pub async fn closed(&mut self) { use crate::future::poll_fn; - poll_fn(|cx| self.poll_closed(cx)).await + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let closed = trace::async_op( + || poll_fn(|cx| self.poll_closed(cx)), + resource_span, + "Sender::closed", + "poll_closed", + false, + ); + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let closed = poll_fn(|cx| self.poll_closed(cx)); + + closed.await } /// Returns `true` if the associated [`Receiver`] handle has been dropped. @@ -674,7 +787,7 @@ impl<T> Sender<T> { /// ``` pub fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll<()> { // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let inner = self.inner.as_ref().unwrap(); @@ -725,6 +838,14 @@ impl<T> Drop for Sender<T> { fn drop(&mut self) { if let Some(inner) = self.inner.as_ref() { inner.complete(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + tx_dropped = true, + tx_dropped.op = "override", + ) + }); } } } @@ -792,6 +913,14 @@ impl<T> Receiver<T> { pub fn close(&mut self) { if let Some(inner) = self.inner.as_ref() { inner.close(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + rx_dropped = true, + rx_dropped.op = "override", + ) + }); } } @@ -804,12 +933,16 @@ impl<T> Receiver<T> { /// This function is useful to call from outside the context of an /// asynchronous task. /// + /// Note that unlike the `poll` method, the `try_recv` method cannot fail + /// spuriously. Any send or close event that happens before this call to + /// `try_recv` will be correctly returned to the caller. + /// /// # Return /// /// - `Ok(T)` if a value is pending in the channel. /// - `Err(TryRecvError::Empty)` if no value has been sent yet. /// - `Err(TryRecvError::Closed)` if the sender has dropped without sending - /// a value. + /// a value, or if the message has already been received. /// /// # Examples /// @@ -869,7 +1002,17 @@ impl<T> Receiver<T> { // `UnsafeCell`. Therefore, it is now safe for us to access the // cell. match unsafe { inner.consume_value() } { - Some(value) => Ok(value), + Some(value) => { + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + value_received = true, + value_received.op = "override", + ) + }); + Ok(value) + } None => Err(TryRecvError::Closed), } } else if state.is_closed() { @@ -885,12 +1028,51 @@ impl<T> Receiver<T> { self.inner = None; result } + + /// Blocking receive to call outside of asynchronous contexts. + /// + /// # Panics + /// + /// This function panics if called within an asynchronous execution + /// context. + /// + /// # Examples + /// + /// ``` + /// use std::thread; + /// use tokio::sync::oneshot; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, rx) = oneshot::channel::<u8>(); + /// + /// let sync_code = thread::spawn(move || { + /// assert_eq!(Ok(10), rx.blocking_recv()); + /// }); + /// + /// let _ = tx.send(10); + /// sync_code.join().unwrap(); + /// } + /// ``` + #[track_caller] + #[cfg(feature = "sync")] + pub fn blocking_recv(self) -> Result<T, RecvError> { + crate::future::block_on(self) + } } impl<T> Drop for Receiver<T> { fn drop(&mut self) { if let Some(inner) = self.inner.as_ref() { inner.close(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + rx_dropped = true, + rx_dropped.op = "override", + ) + }); } } } @@ -900,8 +1082,21 @@ impl<T> Future for Receiver<T> { fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { // If `inner` is `None`, then `poll()` has already completed. + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _res_span = self.resource_span.clone().entered(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _ao_span = self.async_op_span.clone().entered(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _ao_poll_span = self.async_op_poll_span.clone().entered(); + let ret = if let Some(inner) = self.as_ref().get_ref().inner.as_ref() { - ready!(inner.poll_recv(cx))? + #[cfg(all(tokio_unstable, feature = "tracing"))] + let res = ready!(trace_poll_op!("poll_recv", inner.poll_recv(cx)))?; + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + let res = ready!(inner.poll_recv(cx))?; + + res } else { panic!("called after complete"); }; @@ -931,7 +1126,7 @@ impl<T> Inner<T> { fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Result<T, RecvError>> { // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); // Load the state let mut state = State::load(&self.state, Acquire); diff --git a/src/sync/rwlock.rs b/src/sync/rwlock.rs index 120bc72..0492e4e 100644 --- a/src/sync/rwlock.rs +++ b/src/sync/rwlock.rs @@ -1,5 +1,7 @@ use crate::sync::batch_semaphore::{Semaphore, TryAcquireError}; use crate::sync::mutex::TryLockError; +#[cfg(all(tokio_unstable, feature = "tracing"))] +use crate::util::trace; use std::cell::UnsafeCell; use std::marker; use std::marker::PhantomData; @@ -86,6 +88,9 @@ const MAX_READS: u32 = 10; /// [_write-preferring_]: https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock#Priority_policies #[derive(Debug)] pub struct RwLock<T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + // maximum number of concurrent readers mr: u32, @@ -197,14 +202,55 @@ impl<T: ?Sized> RwLock<T> { /// /// let lock = RwLock::new(5); /// ``` + #[track_caller] pub fn new(value: T) -> RwLock<T> where T: Sized, { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let location = std::panic::Location::caller(); + let resource_span = tracing::trace_span!( + "runtime.resource", + concrete_type = "RwLock", + kind = "Sync", + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + ); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + max_readers = MAX_READS, + ); + + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + ); + + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 0, + ); + }); + + resource_span + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let s = resource_span.in_scope(|| Semaphore::new(MAX_READS as usize)); + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + let s = Semaphore::new(MAX_READS as usize); + RwLock { mr: MAX_READS, c: UnsafeCell::new(value), - s: Semaphore::new(MAX_READS as usize), + s, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -222,6 +268,7 @@ impl<T: ?Sized> RwLock<T> { /// # Panics /// /// Panics if `max_reads` is more than `u32::MAX >> 3`. + #[track_caller] pub fn with_max_readers(value: T, max_reads: u32) -> RwLock<T> where T: Sized, @@ -231,10 +278,52 @@ impl<T: ?Sized> RwLock<T> { "a RwLock may not be created with more than {} readers", MAX_READS ); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let location = std::panic::Location::caller(); + + let resource_span = tracing::trace_span!( + "runtime.resource", + concrete_type = "RwLock", + kind = "Sync", + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + ); + + resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + max_readers = max_reads, + ); + + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + ); + + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 0, + ); + }); + + resource_span + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let s = resource_span.in_scope(|| Semaphore::new(max_reads as usize)); + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + let s = Semaphore::new(max_reads as usize); + RwLock { mr: max_reads, c: UnsafeCell::new(value), - s: Semaphore::new(max_reads as usize), + s, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -257,6 +346,8 @@ impl<T: ?Sized> RwLock<T> { mr: MAX_READS, c: UnsafeCell::new(value), s: Semaphore::const_new(MAX_READS as usize), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span::none(), } } @@ -281,6 +372,8 @@ impl<T: ?Sized> RwLock<T> { mr: max_reads, c: UnsafeCell::new(value), s: Semaphore::const_new(max_reads as usize), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span::none(), } } @@ -327,21 +420,98 @@ impl<T: ?Sized> RwLock<T> { /// /// // Drop the guard after the spawned task finishes. /// drop(n); - ///} + /// } /// ``` pub async fn read(&self) -> RwLockReadGuard<'_, T> { - self.s.acquire(1).await.unwrap_or_else(|_| { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let inner = trace::async_op( + || self.s.acquire(1), + self.resource_span.clone(), + "RwLock::read", + "poll", + false, + ); + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let inner = self.s.acquire(1); + + inner.await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + RwLockReadGuard { s: &self.s, data: self.c.get(), marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), } } + /// Blockingly locks this `RwLock` with shared read access. + /// + /// This method is intended for use cases where you + /// need to use this rwlock in asynchronous code as well as in synchronous code. + /// + /// Returns an RAII guard which will drop the read access of this `RwLock` when dropped. + /// + /// # Panics + /// + /// This function panics if called within an asynchronous execution context. + /// + /// - If you find yourself in an asynchronous execution context and needing + /// to call some (synchronous) function which performs one of these + /// `blocking_` operations, then consider wrapping that call inside + /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking] + /// (or [`block_in_place()`][crate::task::block_in_place]). + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// use tokio::sync::RwLock; + /// + /// #[tokio::main] + /// async fn main() { + /// let rwlock = Arc::new(RwLock::new(1)); + /// let mut write_lock = rwlock.write().await; + /// + /// let blocking_task = tokio::task::spawn_blocking({ + /// let rwlock = Arc::clone(&rwlock); + /// move || { + /// // This shall block until the `write_lock` is released. + /// let read_lock = rwlock.blocking_read(); + /// assert_eq!(*read_lock, 0); + /// } + /// }); + /// + /// *write_lock -= 1; + /// drop(write_lock); // release the lock. + /// + /// // Await the completion of the blocking task. + /// blocking_task.await.unwrap(); + /// + /// // Assert uncontended. + /// assert!(rwlock.try_write().is_ok()); + /// } + /// ``` + #[track_caller] + #[cfg(feature = "sync")] + pub fn blocking_read(&self) -> RwLockReadGuard<'_, T> { + crate::future::block_on(self.read()) + } + /// Locks this `RwLock` with shared read access, causing the current task /// to yield until the lock has been acquired. /// @@ -394,15 +564,42 @@ impl<T: ?Sized> RwLock<T> { ///} /// ``` pub async fn read_owned(self: Arc<Self>) -> OwnedRwLockReadGuard<T> { - self.s.acquire(1).await.unwrap_or_else(|_| { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let inner = trace::async_op( + || self.s.acquire(1), + self.resource_span.clone(), + "RwLock::read_owned", + "poll", + false, + ); + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let inner = self.s.acquire(1); + + inner.await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + OwnedRwLockReadGuard { data: self.c.get(), lock: ManuallyDrop::new(self), _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -445,10 +642,21 @@ impl<T: ?Sized> RwLock<T> { Err(TryAcquireError::Closed) => unreachable!(), } + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + Ok(RwLockReadGuard { s: &self.s, data: self.c.get(), marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), }) } @@ -497,10 +705,24 @@ impl<T: ?Sized> RwLock<T> { Err(TryAcquireError::Closed) => unreachable!(), } + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + Ok(OwnedRwLockReadGuard { data: self.c.get(), lock: ManuallyDrop::new(self), _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } @@ -533,19 +755,98 @@ impl<T: ?Sized> RwLock<T> { ///} /// ``` pub async fn write(&self) -> RwLockWriteGuard<'_, T> { - self.s.acquire(self.mr).await.unwrap_or_else(|_| { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let inner = trace::async_op( + || self.s.acquire(self.mr), + self.resource_span.clone(), + "RwLock::write", + "poll", + false, + ); + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let inner = self.s.acquire(self.mr); + + inner.await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = true, + write_locked.op = "override", + ) + }); + RwLockWriteGuard { permits_acquired: self.mr, s: &self.s, data: self.c.get(), marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), } } + /// Blockingly locks this `RwLock` with exclusive write access. + /// + /// This method is intended for use cases where you + /// need to use this rwlock in asynchronous code as well as in synchronous code. + /// + /// Returns an RAII guard which will drop the write access of this `RwLock` when dropped. + /// + /// # Panics + /// + /// This function panics if called within an asynchronous execution context. + /// + /// - If you find yourself in an asynchronous execution context and needing + /// to call some (synchronous) function which performs one of these + /// `blocking_` operations, then consider wrapping that call inside + /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking] + /// (or [`block_in_place()`][crate::task::block_in_place]). + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// use tokio::{sync::RwLock}; + /// + /// #[tokio::main] + /// async fn main() { + /// let rwlock = Arc::new(RwLock::new(1)); + /// let read_lock = rwlock.read().await; + /// + /// let blocking_task = tokio::task::spawn_blocking({ + /// let rwlock = Arc::clone(&rwlock); + /// move || { + /// // This shall block until the `read_lock` is released. + /// let mut write_lock = rwlock.blocking_write(); + /// *write_lock = 2; + /// } + /// }); + /// + /// assert_eq!(*read_lock, 1); + /// // Release the last outstanding read lock. + /// drop(read_lock); + /// + /// // Await the completion of the blocking task. + /// blocking_task.await.unwrap(); + /// + /// // Assert uncontended. + /// let read_lock = rwlock.try_read().unwrap(); + /// assert_eq!(*read_lock, 2); + /// } + /// ``` + #[track_caller] + #[cfg(feature = "sync")] + pub fn blocking_write(&self) -> RwLockWriteGuard<'_, T> { + crate::future::block_on(self.write()) + } + /// Locks this `RwLock` with exclusive write access, causing the current /// task to yield until the lock has been acquired. /// @@ -582,16 +883,43 @@ impl<T: ?Sized> RwLock<T> { ///} /// ``` pub async fn write_owned(self: Arc<Self>) -> OwnedRwLockWriteGuard<T> { - self.s.acquire(self.mr).await.unwrap_or_else(|_| { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let inner = trace::async_op( + || self.s.acquire(self.mr), + self.resource_span.clone(), + "RwLock::write_owned", + "poll", + false, + ); + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let inner = self.s.acquire(self.mr); + + inner.await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = true, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + OwnedRwLockWriteGuard { permits_acquired: self.mr, data: self.c.get(), lock: ManuallyDrop::new(self), _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -625,11 +953,22 @@ impl<T: ?Sized> RwLock<T> { Err(TryAcquireError::Closed) => unreachable!(), } + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = true, + write_locked.op = "override", + ) + }); + Ok(RwLockWriteGuard { permits_acquired: self.mr, s: &self.s, data: self.c.get(), marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), }) } @@ -670,11 +1009,25 @@ impl<T: ?Sized> RwLock<T> { Err(TryAcquireError::Closed) => unreachable!(), } + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = true, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + Ok(OwnedRwLockWriteGuard { permits_acquired: self.mr, data: self.c.get(), lock: ManuallyDrop::new(self), _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } diff --git a/src/sync/rwlock/owned_read_guard.rs b/src/sync/rwlock/owned_read_guard.rs index 1881295..27b71bd 100644 --- a/src/sync/rwlock/owned_read_guard.rs +++ b/src/sync/rwlock/owned_read_guard.rs @@ -15,6 +15,8 @@ use std::sync::Arc; /// [`read_owned`]: method@crate::sync::RwLock::read_owned /// [`RwLock`]: struct@crate::sync::RwLock pub struct OwnedRwLockReadGuard<T: ?Sized, U: ?Sized = T> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) resource_span: tracing::Span, // ManuallyDrop allows us to destructure into this field without running the destructor. pub(super) lock: ManuallyDrop<Arc<RwLock<T>>>, pub(super) data: *const U, @@ -56,12 +58,17 @@ impl<T: ?Sized, U: ?Sized> OwnedRwLockReadGuard<T, U> { { let data = f(&*this) as *const V; let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + OwnedRwLockReadGuard { lock: ManuallyDrop::new(lock), data, _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -105,12 +112,17 @@ impl<T: ?Sized, U: ?Sized> OwnedRwLockReadGuard<T, U> { None => return Err(this), }; let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + Ok(OwnedRwLockReadGuard { lock: ManuallyDrop::new(lock), data, _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } } @@ -145,5 +157,14 @@ impl<T: ?Sized, U: ?Sized> Drop for OwnedRwLockReadGuard<T, U> { fn drop(&mut self) { self.lock.s.release(1); unsafe { ManuallyDrop::drop(&mut self.lock) }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "sub", + ) + }); } } diff --git a/src/sync/rwlock/owned_write_guard.rs b/src/sync/rwlock/owned_write_guard.rs index 0a78d28..dbedab4 100644 --- a/src/sync/rwlock/owned_write_guard.rs +++ b/src/sync/rwlock/owned_write_guard.rs @@ -16,6 +16,8 @@ use std::sync::Arc; /// [`write_owned`]: method@crate::sync::RwLock::write_owned /// [`RwLock`]: struct@crate::sync::RwLock pub struct OwnedRwLockWriteGuard<T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, // ManuallyDrop allows us to destructure into this field without running the destructor. pub(super) lock: ManuallyDrop<Arc<RwLock<T>>>, @@ -64,13 +66,18 @@ impl<T: ?Sized> OwnedRwLockWriteGuard<T> { let data = f(&mut *this) as *mut U; let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; let permits_acquired = this.permits_acquired; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + OwnedRwLockMappedWriteGuard { permits_acquired, lock: ManuallyDrop::new(lock), data, _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -123,13 +130,19 @@ impl<T: ?Sized> OwnedRwLockWriteGuard<T> { }; let permits_acquired = this.permits_acquired; let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); + // NB: Forget to avoid drop impl from being called. mem::forget(this); + Ok(OwnedRwLockMappedWriteGuard { permits_acquired, lock: ManuallyDrop::new(lock), data, _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } @@ -181,15 +194,39 @@ impl<T: ?Sized> OwnedRwLockWriteGuard<T> { pub fn downgrade(mut self) -> OwnedRwLockReadGuard<T> { let lock = unsafe { ManuallyDrop::take(&mut self.lock) }; let data = self.data; + let to_release = (self.permits_acquired - 1) as usize; // Release all but one of the permits held by the write guard - lock.s.release((self.permits_acquired - 1) as usize); + lock.s.release(to_release); + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(self); + OwnedRwLockReadGuard { lock: ManuallyDrop::new(lock), data, _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } } @@ -229,6 +266,14 @@ where impl<T: ?Sized> Drop for OwnedRwLockWriteGuard<T> { fn drop(&mut self) { self.lock.s.release(self.permits_acquired as usize); + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); unsafe { ManuallyDrop::drop(&mut self.lock) }; } } diff --git a/src/sync/rwlock/owned_write_guard_mapped.rs b/src/sync/rwlock/owned_write_guard_mapped.rs index d88ee01..55a24d9 100644 --- a/src/sync/rwlock/owned_write_guard_mapped.rs +++ b/src/sync/rwlock/owned_write_guard_mapped.rs @@ -15,6 +15,8 @@ use std::sync::Arc; /// [mapping]: method@crate::sync::OwnedRwLockWriteGuard::map /// [`OwnedRwLockWriteGuard`]: struct@crate::sync::OwnedRwLockWriteGuard pub struct OwnedRwLockMappedWriteGuard<T: ?Sized, U: ?Sized = T> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, // ManuallyDrop allows us to destructure into this field without running the destructor. pub(super) lock: ManuallyDrop<Arc<RwLock<T>>>, @@ -63,13 +65,18 @@ impl<T: ?Sized, U: ?Sized> OwnedRwLockMappedWriteGuard<T, U> { let data = f(&mut *this) as *mut V; let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; let permits_acquired = this.permits_acquired; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + OwnedRwLockMappedWriteGuard { permits_acquired, lock: ManuallyDrop::new(lock), data, _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -120,13 +127,18 @@ impl<T: ?Sized, U: ?Sized> OwnedRwLockMappedWriteGuard<T, U> { }; let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; let permits_acquired = this.permits_acquired; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + Ok(OwnedRwLockMappedWriteGuard { permits_acquired, lock: ManuallyDrop::new(lock), data, _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } } @@ -166,6 +178,14 @@ where impl<T: ?Sized, U: ?Sized> Drop for OwnedRwLockMappedWriteGuard<T, U> { fn drop(&mut self) { self.lock.s.release(self.permits_acquired as usize); + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); unsafe { ManuallyDrop::drop(&mut self.lock) }; } } diff --git a/src/sync/rwlock/read_guard.rs b/src/sync/rwlock/read_guard.rs index 090b297..f5fc1d6 100644 --- a/src/sync/rwlock/read_guard.rs +++ b/src/sync/rwlock/read_guard.rs @@ -12,7 +12,10 @@ use std::ops; /// /// [`read`]: method@crate::sync::RwLock::read /// [`RwLock`]: struct@crate::sync::RwLock +#[must_use = "if unused the RwLock will immediately unlock"] pub struct RwLockReadGuard<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) resource_span: tracing::Span, pub(super) s: &'a Semaphore, pub(super) data: *const T, pub(super) marker: marker::PhantomData<&'a T>, @@ -59,12 +62,17 @@ impl<'a, T: ?Sized> RwLockReadGuard<'a, T> { { let data = f(&*this) as *const U; let s = this.s; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + RwLockReadGuard { s, data, marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -113,12 +121,17 @@ impl<'a, T: ?Sized> RwLockReadGuard<'a, T> { None => return Err(this), }; let s = this.s; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + Ok(RwLockReadGuard { s, data, marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } } @@ -152,5 +165,14 @@ where impl<'a, T: ?Sized> Drop for RwLockReadGuard<'a, T> { fn drop(&mut self) { self.s.release(1); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "sub", + ) + }); } } diff --git a/src/sync/rwlock/write_guard.rs b/src/sync/rwlock/write_guard.rs index 8c80ee7..cefa183 100644 --- a/src/sync/rwlock/write_guard.rs +++ b/src/sync/rwlock/write_guard.rs @@ -14,7 +14,10 @@ use std::ops; /// /// [`write`]: method@crate::sync::RwLock::write /// [`RwLock`]: struct@crate::sync::RwLock +#[must_use = "if unused the RwLock will immediately unlock"] pub struct RwLockWriteGuard<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, pub(super) s: &'a Semaphore, pub(super) data: *mut T, @@ -66,6 +69,8 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { let data = f(&mut *this) as *mut U; let s = this.s; let permits_acquired = this.permits_acquired; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); RwLockMappedWriteGuard { @@ -73,6 +78,8 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { s, data, marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -129,6 +136,8 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { }; let s = this.s; let permits_acquired = this.permits_acquired; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); Ok(RwLockMappedWriteGuard { @@ -136,6 +145,8 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { s, data, marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } @@ -188,15 +199,38 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { /// [`RwLock`]: struct@crate::sync::RwLock pub fn downgrade(self) -> RwLockReadGuard<'a, T> { let RwLockWriteGuard { s, data, .. } = self; - + let to_release = (self.permits_acquired - 1) as usize; // Release all but one of the permits held by the write guard - s.release((self.permits_acquired - 1) as usize); + s.release(to_release); + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(self); + RwLockReadGuard { s, data, marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } } @@ -236,5 +270,14 @@ where impl<'a, T: ?Sized> Drop for RwLockWriteGuard<'a, T> { fn drop(&mut self) { self.s.release(self.permits_acquired as usize); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); } } diff --git a/src/sync/rwlock/write_guard_mapped.rs b/src/sync/rwlock/write_guard_mapped.rs index 3cf69de..b5c644a 100644 --- a/src/sync/rwlock/write_guard_mapped.rs +++ b/src/sync/rwlock/write_guard_mapped.rs @@ -14,6 +14,8 @@ use std::ops; /// [mapping]: method@crate::sync::RwLockWriteGuard::map /// [`RwLockWriteGuard`]: struct@crate::sync::RwLockWriteGuard pub struct RwLockMappedWriteGuard<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, pub(super) s: &'a Semaphore, pub(super) data: *mut T, @@ -64,13 +66,18 @@ impl<'a, T: ?Sized> RwLockMappedWriteGuard<'a, T> { let data = f(&mut *this) as *mut U; let s = this.s; let permits_acquired = this.permits_acquired; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + RwLockMappedWriteGuard { permits_acquired, s, data, marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -126,13 +133,18 @@ impl<'a, T: ?Sized> RwLockMappedWriteGuard<'a, T> { }; let s = this.s; let permits_acquired = this.permits_acquired; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = this.resource_span.clone(); // NB: Forget to avoid drop impl from being called. mem::forget(this); + Ok(RwLockMappedWriteGuard { permits_acquired, s, data, marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, }) } } @@ -172,5 +184,14 @@ where impl<'a, T: ?Sized> Drop for RwLockMappedWriteGuard<'a, T> { fn drop(&mut self) { self.s.release(self.permits_acquired as usize); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); } } diff --git a/src/sync/semaphore.rs b/src/sync/semaphore.rs index 839b523..6e5a1a8 100644 --- a/src/sync/semaphore.rs +++ b/src/sync/semaphore.rs @@ -1,5 +1,7 @@ use super::batch_semaphore as ll; // low level implementation use super::{AcquireError, TryAcquireError}; +#[cfg(all(tokio_unstable, feature = "tracing"))] +use crate::util::trace; use std::sync::Arc; /// Counting semaphore performing asynchronous permit acquisition. @@ -77,6 +79,8 @@ use std::sync::Arc; pub struct Semaphore { /// The low level semaphore ll_sem: ll::Semaphore, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, } /// A permit from the semaphore. @@ -119,10 +123,41 @@ fn bounds() { } impl Semaphore { + /// The maximum number of permits which a semaphore can hold. It is `usize::MAX >>> 3`. + /// + /// Exceeding this limit typically results in a panic. + pub const MAX_PERMITS: usize = super::batch_semaphore::Semaphore::MAX_PERMITS; + /// Creates a new semaphore with the initial number of permits. + /// + /// Panics if `permits` exceeds [`Semaphore::MAX_PERMITS`]. + #[track_caller] pub fn new(permits: usize) -> Self { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let location = std::panic::Location::caller(); + + tracing::trace_span!( + "runtime.resource", + concrete_type = "Semaphore", + kind = "Sync", + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + inherits_child_attrs = true, + ) + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let ll_sem = resource_span.in_scope(|| ll::Semaphore::new(permits)); + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + let ll_sem = ll::Semaphore::new(permits); + Self { - ll_sem: ll::Semaphore::new(permits), + ll_sem, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -139,9 +174,16 @@ impl Semaphore { #[cfg(all(feature = "parking_lot", not(all(loom, test))))] #[cfg_attr(docsrs, doc(cfg(feature = "parking_lot")))] pub const fn const_new(permits: usize) -> Self { - Self { + #[cfg(all(tokio_unstable, feature = "tracing"))] + return Self { ll_sem: ll::Semaphore::const_new(permits), - } + resource_span: tracing::Span::none(), + }; + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + return Self { + ll_sem: ll::Semaphore::const_new(permits), + }; } /// Returns the current number of available permits. @@ -151,7 +193,7 @@ impl Semaphore { /// Adds `n` new permits to the semaphore. /// - /// The maximum number of permits is `usize::MAX >> 3`, and this function will panic if the limit is exceeded. + /// The maximum number of permits is [`Semaphore::MAX_PERMITS`], and this function will panic if the limit is exceeded. pub fn add_permits(&self, n: usize) { self.ll_sem.release(n); } @@ -191,7 +233,18 @@ impl Semaphore { /// [`AcquireError`]: crate::sync::AcquireError /// [`SemaphorePermit`]: crate::sync::SemaphorePermit pub async fn acquire(&self) -> Result<SemaphorePermit<'_>, AcquireError> { - self.ll_sem.acquire(1).await?; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let inner = trace::async_op( + || self.ll_sem.acquire(1), + self.resource_span.clone(), + "Semaphore::acquire", + "poll", + true, + ); + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let inner = self.ll_sem.acquire(1); + + inner.await?; Ok(SemaphorePermit { sem: self, permits: 1, @@ -227,7 +280,19 @@ impl Semaphore { /// [`AcquireError`]: crate::sync::AcquireError /// [`SemaphorePermit`]: crate::sync::SemaphorePermit pub async fn acquire_many(&self, n: u32) -> Result<SemaphorePermit<'_>, AcquireError> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + trace::async_op( + || self.ll_sem.acquire(n), + self.resource_span.clone(), + "Semaphore::acquire_many", + "poll", + true, + ) + .await?; + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] self.ll_sem.acquire(n).await?; + Ok(SemaphorePermit { sem: self, permits: n, @@ -350,7 +415,18 @@ impl Semaphore { /// [`AcquireError`]: crate::sync::AcquireError /// [`OwnedSemaphorePermit`]: crate::sync::OwnedSemaphorePermit pub async fn acquire_owned(self: Arc<Self>) -> Result<OwnedSemaphorePermit, AcquireError> { - self.ll_sem.acquire(1).await?; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let inner = trace::async_op( + || self.ll_sem.acquire(1), + self.resource_span.clone(), + "Semaphore::acquire_owned", + "poll", + true, + ); + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let inner = self.ll_sem.acquire(1); + + inner.await?; Ok(OwnedSemaphorePermit { sem: self, permits: 1, @@ -403,7 +479,18 @@ impl Semaphore { self: Arc<Self>, n: u32, ) -> Result<OwnedSemaphorePermit, AcquireError> { - self.ll_sem.acquire(n).await?; + #[cfg(all(tokio_unstable, feature = "tracing"))] + let inner = trace::async_op( + || self.ll_sem.acquire(n), + self.resource_span.clone(), + "Semaphore::acquire_many_owned", + "poll", + true, + ); + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let inner = self.ll_sem.acquire(n); + + inner.await?; Ok(OwnedSemaphorePermit { sem: self, permits: n, @@ -540,6 +627,25 @@ impl<'a> SemaphorePermit<'a> { pub fn forget(mut self) { self.permits = 0; } + + /// Merge two [`SemaphorePermit`] instances together, consuming `other` + /// without releasing the permits it holds. + /// + /// Permits held by both `self` and `other` are released when `self` drops. + /// + /// # Panics + /// + /// This function panics if permits from different [`Semaphore`] instances + /// are merged. + #[track_caller] + pub fn merge(&mut self, mut other: Self) { + assert!( + std::ptr::eq(self.sem, other.sem), + "merging permits from different semaphore instances" + ); + self.permits += other.permits; + other.permits = 0; + } } impl OwnedSemaphorePermit { @@ -549,9 +655,28 @@ impl OwnedSemaphorePermit { pub fn forget(mut self) { self.permits = 0; } + + /// Merge two [`OwnedSemaphorePermit`] instances together, consuming `other` + /// without releasing the permits it holds. + /// + /// Permits held by both `self` and `other` are released when `self` drops. + /// + /// # Panics + /// + /// This function panics if permits from different [`Semaphore`] instances + /// are merged. + #[track_caller] + pub fn merge(&mut self, mut other: Self) { + assert!( + Arc::ptr_eq(&self.sem, &other.sem), + "merging permits from different semaphore instances" + ); + self.permits += other.permits; + other.permits = 0; + } } -impl<'a> Drop for SemaphorePermit<'_> { +impl Drop for SemaphorePermit<'_> { fn drop(&mut self) { self.sem.add_permits(self.permits as usize); } diff --git a/src/sync/task/atomic_waker.rs b/src/sync/task/atomic_waker.rs index e1330fb..13aba35 100644 --- a/src/sync/task/atomic_waker.rs +++ b/src/sync/task/atomic_waker.rs @@ -1,7 +1,8 @@ #![cfg_attr(any(loom, not(feature = "sync")), allow(dead_code, unreachable_pub))] use crate::loom::cell::UnsafeCell; -use crate::loom::sync::atomic::{self, AtomicUsize}; +use crate::loom::hint; +use crate::loom::sync::atomic::AtomicUsize; use std::fmt; use std::panic::{resume_unwind, AssertUnwindSafe, RefUnwindSafe, UnwindSafe}; @@ -281,9 +282,7 @@ impl AtomicWaker { waker.wake(); // This is equivalent to a spin lock, so use a spin hint. - // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead. - #[allow(deprecated)] - atomic::spin_loop_hint(); + hint::spin_loop(); } state => { // In this case, a concurrent thread is holding the diff --git a/src/sync/tests/atomic_waker.rs b/src/sync/tests/atomic_waker.rs index b167a5d..8ebfb91 100644 --- a/src/sync/tests/atomic_waker.rs +++ b/src/sync/tests/atomic_waker.rs @@ -4,7 +4,7 @@ use tokio_test::task; use std::task::Waker; trait AssertSend: Send {} -trait AssertSync: Send {} +trait AssertSync: Sync {} impl AssertSend for AtomicWaker {} impl AssertSync for AtomicWaker {} @@ -12,6 +12,9 @@ impl AssertSync for AtomicWaker {} impl AssertSend for Waker {} impl AssertSync for Waker {} +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; + #[test] fn basic_usage() { let mut waker = task::spawn(AtomicWaker::new()); @@ -34,6 +37,7 @@ fn wake_without_register() { } #[test] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn atomic_waker_panic_safe() { use std::panic; use std::ptr; diff --git a/src/sync/tests/notify.rs b/src/sync/tests/notify.rs index 8c9a573..eb0da8f 100644 --- a/src/sync/tests/notify.rs +++ b/src/sync/tests/notify.rs @@ -4,6 +4,9 @@ use std::mem::ManuallyDrop; use std::sync::Arc; use std::task::{Context, RawWaker, RawWakerVTable, Waker}; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; + #[test] fn notify_clones_waker_before_lock() { const VTABLE: &RawWakerVTable = &RawWakerVTable::new(clone_w, wake, wake_by_ref, drop_w); @@ -42,3 +45,37 @@ fn notify_clones_waker_before_lock() { // The result doesn't matter, we're just testing that we don't deadlock. let _ = future.poll(&mut cx); } + +#[test] +fn notify_simple() { + let notify = Notify::new(); + + let mut fut1 = tokio_test::task::spawn(notify.notified()); + assert!(fut1.poll().is_pending()); + + let mut fut2 = tokio_test::task::spawn(notify.notified()); + assert!(fut2.poll().is_pending()); + + notify.notify_waiters(); + + assert!(fut1.poll().is_ready()); + assert!(fut2.poll().is_ready()); +} + +#[test] +#[cfg(not(tokio_wasm))] +fn watch_test() { + let rt = crate::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(async { + let (tx, mut rx) = crate::sync::watch::channel(()); + + crate::spawn(async move { + let _ = tx.send(()); + }); + + let _ = rx.changed().await; + }); +} diff --git a/src/sync/tests/semaphore_batch.rs b/src/sync/tests/semaphore_batch.rs index 9342cd1..d4e35aa 100644 --- a/src/sync/tests/semaphore_batch.rs +++ b/src/sync/tests/semaphore_batch.rs @@ -1,6 +1,9 @@ use crate::sync::batch_semaphore::Semaphore; use tokio_test::*; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; + #[test] fn poll_acquire_one_available() { let s = Semaphore::new(100); @@ -167,6 +170,7 @@ fn poll_acquire_one_zero_permits() { #[test] #[should_panic] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn validates_max_permits() { use std::usize; Semaphore::new((usize::MAX >> 2) + 1); diff --git a/src/sync/watch.rs b/src/sync/watch.rs index 7e45c11..6db46b7 100644 --- a/src/sync/watch.rs +++ b/src/sync/watch.rs @@ -9,10 +9,10 @@ //! # Usage //! //! [`channel`] returns a [`Sender`] / [`Receiver`] pair. These are the producer -//! and sender halves of the channel. The channel is created with an initial +//! and consumer halves of the channel. The channel is created with an initial //! value. The **latest** value stored in the channel is accessed with //! [`Receiver::borrow()`]. Awaiting [`Receiver::changed()`] waits for a new -//! value to sent by the [`Sender`] half. +//! value to be sent by the [`Sender`] half. //! //! # Examples //! @@ -20,15 +20,15 @@ //! use tokio::sync::watch; //! //! # async fn dox() -> Result<(), Box<dyn std::error::Error>> { -//! let (tx, mut rx) = watch::channel("hello"); +//! let (tx, mut rx) = watch::channel("hello"); //! -//! tokio::spawn(async move { -//! while rx.changed().await.is_ok() { -//! println!("received = {:?}", *rx.borrow()); -//! } -//! }); +//! tokio::spawn(async move { +//! while rx.changed().await.is_ok() { +//! println!("received = {:?}", *rx.borrow()); +//! } +//! }); //! -//! tx.send("world")?; +//! tx.send("world")?; //! # Ok(()) //! # } //! ``` @@ -60,6 +60,7 @@ use crate::loom::sync::atomic::Ordering::Relaxed; use crate::loom::sync::{Arc, RwLock, RwLockReadGuard}; use std::mem; use std::ops; +use std::panic; /// Receives values from the associated [`Sender`](struct@Sender). /// @@ -89,11 +90,80 @@ pub struct Sender<T> { /// Returns a reference to the inner value. /// /// Outstanding borrows hold a read lock on the inner value. This means that -/// long lived borrows could cause the produce half to block. It is recommended -/// to keep the borrow as short lived as possible. +/// long-lived borrows could cause the producer half to block. It is recommended +/// to keep the borrow as short-lived as possible. Additionally, if you are +/// running in an environment that allows `!Send` futures, you must ensure that +/// the returned `Ref` type is never held alive across an `.await` point, +/// otherwise, it can lead to a deadlock. +/// +/// The priority policy of the lock is dependent on the underlying lock +/// implementation, and this type does not guarantee that any particular policy +/// will be used. In particular, a producer which is waiting to acquire the lock +/// in `send` might or might not block concurrent calls to `borrow`, e.g.: +/// +/// <details><summary>Potential deadlock example</summary> +/// +/// ```text +/// // Task 1 (on thread A) | // Task 2 (on thread B) +/// let _ref1 = rx.borrow(); | +/// | // will block +/// | let _ = tx.send(()); +/// // may deadlock | +/// let _ref2 = rx.borrow(); | +/// ``` +/// </details> #[derive(Debug)] pub struct Ref<'a, T> { inner: RwLockReadGuard<'a, T>, + has_changed: bool, +} + +impl<'a, T> Ref<'a, T> { + /// Indicates if the borrowed value is considered as _changed_ since the last + /// time it has been marked as seen. + /// + /// Unlike [`Receiver::has_changed()`], this method does not fail if the channel is closed. + /// + /// When borrowed from the [`Sender`] this function will always return `false`. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx) = watch::channel("hello"); + /// + /// tx.send("goodbye").unwrap(); + /// // The sender does never consider the value as changed. + /// assert!(!tx.borrow().has_changed()); + /// + /// // Drop the sender immediately, just for testing purposes. + /// drop(tx); + /// + /// // Even if the sender has already been dropped... + /// assert!(rx.has_changed().is_err()); + /// // ...the modified value is still readable and detected as changed. + /// assert_eq!(*rx.borrow(), "goodbye"); + /// assert!(rx.borrow().has_changed()); + /// + /// // Read the changed value and mark it as seen. + /// { + /// let received = rx.borrow_and_update(); + /// assert_eq!(*received, "goodbye"); + /// assert!(received.has_changed()); + /// // Release the read lock when leaving this scope. + /// } + /// + /// // Now the value has already been marked as seen and could + /// // never be modified again (after the sender has been dropped). + /// assert!(!rx.borrow().has_changed()); + /// } + /// ``` + pub fn has_changed(&self) -> bool { + self.has_changed + } } #[derive(Debug)] @@ -137,7 +207,7 @@ pub mod error { impl<T: fmt::Debug> std::error::Error for SendError<T> {} /// Error produced when receiving a change notification. - #[derive(Debug)] + #[derive(Debug, Clone)] pub struct RecvError(pub(super) ()); // ===== impl RecvError ===== @@ -281,9 +351,29 @@ impl<T> Receiver<T> { /// [`changed`] may return immediately even if you have already seen the /// value with a call to `borrow`. /// - /// Outstanding borrows hold a read lock. This means that long lived borrows - /// could cause the send half to block. It is recommended to keep the borrow - /// as short lived as possible. + /// Outstanding borrows hold a read lock on the inner value. This means that + /// long-lived borrows could cause the producer half to block. It is recommended + /// to keep the borrow as short-lived as possible. Additionally, if you are + /// running in an environment that allows `!Send` futures, you must ensure that + /// the returned `Ref` type is never held alive across an `.await` point, + /// otherwise, it can lead to a deadlock. + /// + /// The priority policy of the lock is dependent on the underlying lock + /// implementation, and this type does not guarantee that any particular policy + /// will be used. In particular, a producer which is waiting to acquire the lock + /// in `send` might or might not block concurrent calls to `borrow`, e.g.: + /// + /// <details><summary>Potential deadlock example</summary> + /// + /// ```text + /// // Task 1 (on thread A) | // Task 2 (on thread B) + /// let _ref1 = rx.borrow(); | + /// | // will block + /// | let _ = tx.send(()); + /// // may deadlock | + /// let _ref2 = rx.borrow(); | + /// ``` + /// </details> /// /// [`changed`]: Receiver::changed /// @@ -297,25 +387,101 @@ impl<T> Receiver<T> { /// ``` pub fn borrow(&self) -> Ref<'_, T> { let inner = self.shared.value.read().unwrap(); - Ref { inner } + + // After obtaining a read-lock no concurrent writes could occur + // and the loaded version matches that of the borrowed reference. + let new_version = self.shared.state.load().version(); + let has_changed = self.version != new_version; + + Ref { inner, has_changed } } - /// Returns a reference to the most recently sent value and mark that value + /// Returns a reference to the most recently sent value and marks that value /// as seen. /// - /// This method marks the value as seen, so [`changed`] will not return - /// immediately if the newest value is one previously returned by - /// `borrow_and_update`. + /// This method marks the current value as seen. Subsequent calls to [`changed`] + /// will not return immediately until the [`Sender`] has modified the shared + /// value again. + /// + /// Outstanding borrows hold a read lock on the inner value. This means that + /// long-lived borrows could cause the producer half to block. It is recommended + /// to keep the borrow as short-lived as possible. Additionally, if you are + /// running in an environment that allows `!Send` futures, you must ensure that + /// the returned `Ref` type is never held alive across an `.await` point, + /// otherwise, it can lead to a deadlock. + /// + /// The priority policy of the lock is dependent on the underlying lock + /// implementation, and this type does not guarantee that any particular policy + /// will be used. In particular, a producer which is waiting to acquire the lock + /// in `send` might or might not block concurrent calls to `borrow`, e.g.: + /// + /// <details><summary>Potential deadlock example</summary> /// - /// Outstanding borrows hold a read lock. This means that long lived borrows - /// could cause the send half to block. It is recommended to keep the borrow - /// as short lived as possible. + /// ```text + /// // Task 1 (on thread A) | // Task 2 (on thread B) + /// let _ref1 = rx1.borrow_and_update(); | + /// | // will block + /// | let _ = tx.send(()); + /// // may deadlock | + /// let _ref2 = rx2.borrow_and_update(); | + /// ``` + /// </details> /// /// [`changed`]: Receiver::changed pub fn borrow_and_update(&mut self) -> Ref<'_, T> { let inner = self.shared.value.read().unwrap(); - self.version = self.shared.state.load().version(); - Ref { inner } + + // After obtaining a read-lock no concurrent writes could occur + // and the loaded version matches that of the borrowed reference. + let new_version = self.shared.state.load().version(); + let has_changed = self.version != new_version; + + // Mark the shared value as seen by updating the version + self.version = new_version; + + Ref { inner, has_changed } + } + + /// Checks if this channel contains a message that this receiver has not yet + /// seen. The new value is not marked as seen. + /// + /// Although this method is called `has_changed`, it does not check new + /// messages for equality, so this call will return true even if the new + /// message is equal to the old message. + /// + /// Returns an error if the channel has been closed. + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx) = watch::channel("hello"); + /// + /// tx.send("goodbye").unwrap(); + /// + /// assert!(rx.has_changed().unwrap()); + /// assert_eq!(*rx.borrow_and_update(), "goodbye"); + /// + /// // The value has been marked as seen + /// assert!(!rx.has_changed().unwrap()); + /// + /// drop(tx); + /// // The `tx` handle has been dropped + /// assert!(rx.has_changed().is_err()); + /// } + /// ``` + pub fn has_changed(&self) -> Result<bool, error::RecvError> { + // Load the version from the state + let state = self.shared.state.load(); + if state.is_closed() { + // The sender has dropped. + return Err(error::RecvError(())); + } + let new_version = state.version(); + + Ok(self.version != new_version) } /// Waits for a change notification, then marks the newest value as seen. @@ -373,6 +539,22 @@ impl<T> Receiver<T> { } } + /// Returns `true` if receivers belong to the same channel. + /// + /// # Examples + /// + /// ``` + /// let (tx, rx) = tokio::sync::watch::channel(true); + /// let rx2 = rx.clone(); + /// assert!(rx.same_channel(&rx2)); + /// + /// let (tx3, rx3) = tokio::sync::watch::channel(true); + /// assert!(!rx3.same_channel(&rx2)); + /// ``` + pub fn same_channel(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.shared, &other.shared) + } + cfg_process_driver! { pub(crate) fn try_has_changed(&mut self) -> Option<Result<(), error::RecvError>> { maybe_changed(&self.shared, &mut self.version) @@ -425,8 +607,22 @@ impl<T> Drop for Receiver<T> { impl<T> Sender<T> { /// Sends a new value via the channel, notifying all receivers. /// - /// This method fails if the channel has been closed, which happens when - /// every receiver has been dropped. + /// This method fails if the channel is closed, which is the case when + /// every receiver has been dropped. It is possible to reopen the channel + /// using the [`subscribe`] method. However, when `send` fails, the value + /// isn't made available for future receivers (but returned with the + /// [`SendError`]). + /// + /// To always make a new value available for future receivers, even if no + /// receiver currently exists, one of the other send methods + /// ([`send_if_modified`], [`send_modify`], or [`send_replace`]) can be + /// used instead. + /// + /// [`subscribe`]: Sender::subscribe + /// [`SendError`]: error::SendError + /// [`send_if_modified`]: Sender::send_if_modified + /// [`send_modify`]: Sender::send_modify + /// [`send_replace`]: Sender::send_replace pub fn send(&self, value: T) -> Result<(), error::SendError<T>> { // This is pretty much only useful as a hint anyway, so synchronization isn't critical. if 0 == self.receiver_count() { @@ -437,27 +633,129 @@ impl<T> Sender<T> { Ok(()) } - /// Sends a new value via the channel, notifying all receivers and returning - /// the previous value in the channel. + /// Modifies the watched value **unconditionally** in-place, + /// notifying all receivers. /// - /// This can be useful for reusing the buffers inside a watched value. - /// Additionally, this method permits sending values even when there are no - /// receivers. + /// This can useful for modifying the watched value, without + /// having to allocate a new instance. Additionally, this + /// method permits sending values even when there are no receivers. + /// + /// Prefer to use the more versatile function [`Self::send_if_modified()`] + /// if the value is only modified conditionally during the mutable borrow + /// to prevent unneeded change notifications for unmodified values. + /// + /// # Panics + /// + /// This function panics when the invocation of the `modify` closure panics. + /// No receivers are notified when panicking. All changes of the watched + /// value applied by the closure before panicking will be visible in + /// subsequent calls to `borrow`. /// /// # Examples /// /// ``` /// use tokio::sync::watch; /// - /// let (tx, _rx) = watch::channel(1); - /// assert_eq!(tx.send_replace(2), 1); - /// assert_eq!(tx.send_replace(3), 2); + /// struct State { + /// counter: usize, + /// } + /// let (state_tx, state_rx) = watch::channel(State { counter: 0 }); + /// state_tx.send_modify(|state| state.counter += 1); + /// assert_eq!(state_rx.borrow().counter, 1); + /// ``` + pub fn send_modify<F>(&self, modify: F) + where + F: FnOnce(&mut T), + { + self.send_if_modified(|value| { + modify(value); + true + }); + } + + /// Modifies the watched value **conditionally** in-place, + /// notifying all receivers only if modified. + /// + /// This can useful for modifying the watched value, without + /// having to allocate a new instance. Additionally, this + /// method permits sending values even when there are no receivers. + /// + /// The `modify` closure must return `true` if the value has actually + /// been modified during the mutable borrow. It should only return `false` + /// if the value is guaranteed to be unmodified despite the mutable + /// borrow. + /// + /// Receivers are only notified if the closure returned `true`. If the + /// closure has modified the value but returned `false` this results + /// in a *silent modification*, i.e. the modified value will be visible + /// in subsequent calls to `borrow`, but receivers will not receive + /// a change notification. + /// + /// Returns the result of the closure, i.e. `true` if the value has + /// been modified and `false` otherwise. + /// + /// # Panics + /// + /// This function panics when the invocation of the `modify` closure panics. + /// No receivers are notified when panicking. All changes of the watched + /// value applied by the closure before panicking will be visible in + /// subsequent calls to `borrow`. + /// + /// # Examples + /// /// ``` - pub fn send_replace(&self, value: T) -> T { - let old = { + /// use tokio::sync::watch; + /// + /// struct State { + /// counter: usize, + /// } + /// let (state_tx, mut state_rx) = watch::channel(State { counter: 1 }); + /// let inc_counter_if_odd = |state: &mut State| { + /// if state.counter % 2 == 1 { + /// state.counter += 1; + /// return true; + /// } + /// false + /// }; + /// + /// assert_eq!(state_rx.borrow().counter, 1); + /// + /// assert!(!state_rx.has_changed().unwrap()); + /// assert!(state_tx.send_if_modified(inc_counter_if_odd)); + /// assert!(state_rx.has_changed().unwrap()); + /// assert_eq!(state_rx.borrow_and_update().counter, 2); + /// + /// assert!(!state_rx.has_changed().unwrap()); + /// assert!(!state_tx.send_if_modified(inc_counter_if_odd)); + /// assert!(!state_rx.has_changed().unwrap()); + /// assert_eq!(state_rx.borrow_and_update().counter, 2); + /// ``` + pub fn send_if_modified<F>(&self, modify: F) -> bool + where + F: FnOnce(&mut T) -> bool, + { + { // Acquire the write lock and update the value. let mut lock = self.shared.value.write().unwrap(); - let old = mem::replace(&mut *lock, value); + + // Update the value and catch possible panic inside func. + let result = panic::catch_unwind(panic::AssertUnwindSafe(|| modify(&mut lock))); + match result { + Ok(modified) => { + if !modified { + // Abort, i.e. don't notify receivers if unmodified + return false; + } + // Continue if modified + } + Err(panicked) => { + // Drop the lock to avoid poisoning it. + drop(lock); + // Forward the panic to the caller. + panic::resume_unwind(panicked); + // Unreachable + } + }; self.shared.state.increment_version(); @@ -467,21 +765,44 @@ impl<T> Sender<T> { // that receivers are able to figure out the version number of the // value they are currently looking at. drop(lock); + } - old - }; - - // Notify all watchers self.shared.notify_rx.notify_waiters(); - old + true + } + + /// Sends a new value via the channel, notifying all receivers and returning + /// the previous value in the channel. + /// + /// This can be useful for reusing the buffers inside a watched value. + /// Additionally, this method permits sending values even when there are no + /// receivers. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// let (tx, _rx) = watch::channel(1); + /// assert_eq!(tx.send_replace(2), 1); + /// assert_eq!(tx.send_replace(3), 2); + /// ``` + pub fn send_replace(&self, mut value: T) -> T { + // swap old watched value with the new one + self.send_modify(|old| mem::swap(old, &mut value)); + + value } /// Returns a reference to the most recently sent value /// - /// Outstanding borrows hold a read lock. This means that long lived borrows - /// could cause the send half to block. It is recommended to keep the borrow - /// as short lived as possible. + /// Outstanding borrows hold a read lock on the inner value. This means that + /// long-lived borrows could cause the producer half to block. It is recommended + /// to keep the borrow as short-lived as possible. Additionally, if you are + /// running in an environment that allows `!Send` futures, you must ensure that + /// the returned `Ref` type is never held alive across an `.await` point, + /// otherwise, it can lead to a deadlock. /// /// # Examples /// @@ -493,7 +814,11 @@ impl<T> Sender<T> { /// ``` pub fn borrow(&self) -> Ref<'_, T> { let inner = self.shared.value.read().unwrap(); - Ref { inner } + + // The sender/producer always sees the current version + let has_changed = false; + + Ref { inner, has_changed } } /// Checks if the channel has been closed. This happens when all receivers diff --git a/src/task/blocking.rs b/src/task/blocking.rs index 825f25f..46756a9 100644 --- a/src/task/blocking.rs +++ b/src/task/blocking.rs @@ -70,11 +70,12 @@ cfg_rt_multi_thread! { /// This function panics if called from a [`current_thread`] runtime. /// /// [`current_thread`]: fn@crate::runtime::Builder::new_current_thread + #[track_caller] pub fn block_in_place<F, R>(f: F) -> R where F: FnOnce() -> R, { - crate::runtime::thread_pool::block_in_place(f) + crate::runtime::scheduler::multi_thread::block_in_place(f) } } @@ -102,14 +103,24 @@ cfg_rt! { /// their own. If you want to spawn an ordinary thread, you should use /// [`thread::spawn`] instead. /// - /// Closures spawned using `spawn_blocking` cannot be cancelled. When you shut - /// down the executor, it will wait indefinitely for all blocking operations to + /// Closures spawned using `spawn_blocking` cannot be cancelled abruptly; there + /// is no standard low level API to cause a thread to stop running. However, + /// a useful pattern is to pass some form of "cancellation token" into + /// the thread. This could be an [`AtomicBool`] that the task checks periodically. + /// Another approach is to have the thread primarily read or write from a channel, + /// and to exit when the channel closes; assuming the other side of the channel is dropped + /// when cancellation occurs, this will cause the blocking task thread to exit + /// soon after as well. + /// + /// When you shut down the executor, it will wait indefinitely for all blocking operations to /// finish. You can use [`shutdown_timeout`] to stop waiting for them after a /// certain timeout. Be aware that this will still not cancel the tasks — they - /// are simply allowed to keep running after the method returns. + /// are simply allowed to keep running after the method returns. It is possible + /// for a blocking task to be cancelled if it has not yet started running, but this + /// is not guaranteed. /// /// Note that if you are using the single threaded runtime, this function will - /// still spawn additional threads for blocking operations. The basic + /// still spawn additional threads for blocking operations. The current-thread /// scheduler's single thread is only used for asynchronous code. /// /// # Related APIs and patterns for bridging asynchronous and blocking code @@ -140,6 +151,7 @@ cfg_rt! { /// [`thread::spawn`]: fn@std::thread::spawn /// [`shutdown_timeout`]: fn@crate::runtime::Runtime::shutdown_timeout /// [bridgesync]: https://tokio.rs/tokio/topics/bridging + /// [`AtomicBool`]: struct@std::sync::atomic::AtomicBool /// /// # Examples /// @@ -188,7 +200,7 @@ cfg_rt! { /// worker.await.unwrap(); /// # } /// ``` - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub fn spawn_blocking<F, R>(f: F) -> JoinHandle<R> where F: FnOnce() -> R + Send + 'static, diff --git a/src/task/builder.rs b/src/task/builder.rs index f991fc6..91c400c 100644 --- a/src/task/builder.rs +++ b/src/task/builder.rs @@ -1,9 +1,16 @@ #![allow(unreachable_pub)] -use crate::{runtime::context, task::JoinHandle}; -use std::future::Future; +use crate::{ + runtime::Handle, + task::{JoinHandle, LocalSet}, +}; +use std::{future::Future, io}; /// Factory which is used to configure the properties of a new task. /// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// /// Methods can be chained in order to configure it. /// /// Currently, there is only one configuration option: @@ -41,11 +48,17 @@ use std::future::Future; /// .spawn(async move { /// // Process each socket concurrently. /// process(socket).await -/// }); +/// })?; /// } /// } /// ``` +/// [unstable]: crate#unstable-features +/// [`name`]: Builder::name +/// [`spawn_local`]: Builder::spawn_local +/// [`spawn`]: Builder::spawn +/// [`spawn_blocking`]: Builder::spawn_blocking #[derive(Default, Debug)] +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] pub struct Builder<'a> { name: Option<&'a str>, } @@ -61,42 +74,128 @@ impl<'a> Builder<'a> { Self { name: Some(name) } } - /// Spawns a task on the executor. + /// Spawns a task with this builder's settings on the current runtime. + /// + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. /// /// See [`task::spawn`](crate::task::spawn) for /// more details. - #[cfg_attr(tokio_track_caller, track_caller)] - pub fn spawn<Fut>(self, future: Fut) -> JoinHandle<Fut::Output> + #[track_caller] + pub fn spawn<Fut>(self, future: Fut) -> io::Result<JoinHandle<Fut::Output>> where Fut: Future + Send + 'static, Fut::Output: Send + 'static, { - super::spawn::spawn_inner(future, self.name) + Ok(super::spawn::spawn_inner(future, self.name)) } - /// Spawns a task on the current thread. + /// Spawn a task with this builder's settings on the provided [runtime + /// handle]. /// - /// See [`task::spawn_local`](crate::task::spawn_local) - /// for more details. - #[cfg_attr(tokio_track_caller, track_caller)] - pub fn spawn_local<Fut>(self, future: Fut) -> JoinHandle<Fut::Output> + /// See [`Handle::spawn`] for more details. + /// + /// [runtime handle]: crate::runtime::Handle + /// [`Handle::spawn`]: crate::runtime::Handle::spawn + #[track_caller] + pub fn spawn_on<Fut>(self, future: Fut, handle: &Handle) -> io::Result<JoinHandle<Fut::Output>> + where + Fut: Future + Send + 'static, + Fut::Output: Send + 'static, + { + Ok(handle.spawn_named(future, self.name)) + } + + /// Spawns `!Send` a task on the current [`LocalSet`] with this builder's + /// settings. + /// + /// The spawned future will be run on the same thread that called `spawn_local`. + /// This may only be called from the context of a [local task set][`LocalSet`]. + /// + /// # Panics + /// + /// This function panics if called outside of a [local task set][`LocalSet`]. + /// + /// See [`task::spawn_local`] for more details. + /// + /// [`task::spawn_local`]: crate::task::spawn_local + /// [`LocalSet`]: crate::task::LocalSet + #[track_caller] + pub fn spawn_local<Fut>(self, future: Fut) -> io::Result<JoinHandle<Fut::Output>> where Fut: Future + 'static, Fut::Output: 'static, { - super::local::spawn_local_inner(future, self.name) + Ok(super::local::spawn_local_inner(future, self.name)) + } + + /// Spawns `!Send` a task on the provided [`LocalSet`] with this builder's + /// settings. + /// + /// See [`LocalSet::spawn_local`] for more details. + /// + /// [`LocalSet::spawn_local`]: crate::task::LocalSet::spawn_local + /// [`LocalSet`]: crate::task::LocalSet + #[track_caller] + pub fn spawn_local_on<Fut>( + self, + future: Fut, + local_set: &LocalSet, + ) -> io::Result<JoinHandle<Fut::Output>> + where + Fut: Future + 'static, + Fut::Output: 'static, + { + Ok(local_set.spawn_named(future, self.name)) } /// Spawns blocking code on the blocking threadpool. /// + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. + /// /// See [`task::spawn_blocking`](crate::task::spawn_blocking) /// for more details. - #[cfg_attr(tokio_track_caller, track_caller)] - pub fn spawn_blocking<Function, Output>(self, function: Function) -> JoinHandle<Output> + #[track_caller] + pub fn spawn_blocking<Function, Output>( + self, + function: Function, + ) -> io::Result<JoinHandle<Output>> where Function: FnOnce() -> Output + Send + 'static, Output: Send + 'static, { - context::current().spawn_blocking_inner(function, self.name) + let handle = Handle::current(); + self.spawn_blocking_on(function, &handle) + } + + /// Spawns blocking code on the provided [runtime handle]'s blocking threadpool. + /// + /// See [`Handle::spawn_blocking`] for more details. + /// + /// [runtime handle]: crate::runtime::Handle + /// [`Handle::spawn_blocking`]: crate::runtime::Handle::spawn_blocking + #[track_caller] + pub fn spawn_blocking_on<Function, Output>( + self, + function: Function, + handle: &Handle, + ) -> io::Result<JoinHandle<Output>> + where + Function: FnOnce() -> Output + Send + 'static, + Output: Send + 'static, + { + use crate::runtime::Mandatory; + let (join_handle, spawn_result) = handle.inner.blocking_spawner().spawn_blocking_inner( + function, + Mandatory::NonMandatory, + self.name, + handle, + ); + + spawn_result?; + Ok(join_handle) } } diff --git a/src/task/consume_budget.rs b/src/task/consume_budget.rs new file mode 100644 index 0000000..1212cfc --- /dev/null +++ b/src/task/consume_budget.rs @@ -0,0 +1,45 @@ +use std::task::Poll; + +/// Consumes a unit of budget and returns the execution back to the Tokio +/// runtime *if* the task's coop budget was exhausted. +/// +/// The task will only yield if its entire coop budget has been exhausted. +/// This function can be used in order to insert optional yield points into long +/// computations that do not use Tokio resources like sockets or semaphores, +/// without redundantly yielding to the runtime each time. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// # Examples +/// +/// Make sure that a function which returns a sum of (potentially lots of) +/// iterated values is cooperative. +/// +/// ``` +/// async fn sum_iterator(input: &mut impl std::iter::Iterator<Item=i64>) -> i64 { +/// let mut sum: i64 = 0; +/// while let Some(i) = input.next() { +/// sum += i; +/// tokio::task::consume_budget().await +/// } +/// sum +/// } +/// ``` +/// [unstable]: crate#unstable-features +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "rt"))))] +pub async fn consume_budget() { + let mut status = Poll::Pending; + + crate::future::poll_fn(move |cx| { + if status.is_ready() { + return status; + } + status = crate::runtime::coop::poll_proceed(cx).map(|restore| { + restore.made_progress(); + }); + status + }) + .await +} diff --git a/src/task/join_set.rs b/src/task/join_set.rs new file mode 100644 index 0000000..e6d8d62 --- /dev/null +++ b/src/task/join_set.rs @@ -0,0 +1,522 @@ +//! A collection of tasks spawned on a Tokio runtime. +//! +//! This module provides the [`JoinSet`] type, a collection which stores a set +//! of spawned tasks and allows asynchronously awaiting the output of those +//! tasks as they complete. See the documentation for the [`JoinSet`] type for +//! details. +use std::fmt; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use crate::runtime::Handle; +#[cfg(tokio_unstable)] +use crate::task::Id; +use crate::task::{AbortHandle, JoinError, JoinHandle, LocalSet}; +use crate::util::IdleNotifiedSet; + +/// A collection of tasks spawned on a Tokio runtime. +/// +/// A `JoinSet` can be used to await the completion of some or all of the tasks +/// in the set. The set is not ordered, and the tasks will be returned in the +/// order they complete. +/// +/// All of the tasks must have the same return type `T`. +/// +/// When the `JoinSet` is dropped, all tasks in the `JoinSet` are immediately aborted. +/// +/// # Examples +/// +/// Spawn multiple tasks and wait for them. +/// +/// ``` +/// use tokio::task::JoinSet; +/// +/// #[tokio::main] +/// async fn main() { +/// let mut set = JoinSet::new(); +/// +/// for i in 0..10 { +/// set.spawn(async move { i }); +/// } +/// +/// let mut seen = [false; 10]; +/// while let Some(res) = set.join_next().await { +/// let idx = res.unwrap(); +/// seen[idx] = true; +/// } +/// +/// for i in 0..10 { +/// assert!(seen[i]); +/// } +/// } +/// ``` +#[cfg_attr(docsrs, doc(cfg(feature = "rt")))] +pub struct JoinSet<T> { + inner: IdleNotifiedSet<JoinHandle<T>>, +} + +/// A variant of [`task::Builder`] that spawns tasks on a [`JoinSet`] rather +/// than on the current default runtime. +/// +/// [`task::Builder`]: crate::task::Builder +#[cfg(all(tokio_unstable, feature = "tracing"))] +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] +#[must_use = "builders do nothing unless used to spawn a task"] +pub struct Builder<'a, T> { + joinset: &'a mut JoinSet<T>, + builder: super::Builder<'a>, +} + +impl<T> JoinSet<T> { + /// Create a new `JoinSet`. + pub fn new() -> Self { + Self { + inner: IdleNotifiedSet::new(), + } + } + + /// Returns the number of tasks currently in the `JoinSet`. + pub fn len(&self) -> usize { + self.inner.len() + } + + /// Returns whether the `JoinSet` is empty. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } +} + +impl<T: 'static> JoinSet<T> { + /// Returns a [`Builder`] that can be used to configure a task prior to + /// spawning it on this `JoinSet`. + /// + /// # Examples + /// + /// ``` + /// use tokio::task::JoinSet; + /// + /// #[tokio::main] + /// async fn main() -> std::io::Result<()> { + /// let mut set = JoinSet::new(); + /// + /// // Use the builder to configure a task's name before spawning it. + /// set.build_task() + /// .name("my_task") + /// .spawn(async { /* ... */ })?; + /// + /// Ok(()) + /// } + /// ``` + #[cfg(all(tokio_unstable, feature = "tracing"))] + #[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] + pub fn build_task(&mut self) -> Builder<'_, T> { + Builder { + builder: super::Builder::new(), + joinset: self, + } + } + + /// Spawn the provided task on the `JoinSet`, returning an [`AbortHandle`] + /// that can be used to remotely cancel the task. + /// + /// The provided future will start running in the background immediately + /// when this method is called, even if you don't await anything on this + /// `JoinSet`. + /// + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn<F>(&mut self, task: F) -> AbortHandle + where + F: Future<Output = T>, + F: Send + 'static, + T: Send, + { + self.insert(crate::spawn(task)) + } + + /// Spawn the provided task on the provided runtime and store it in this + /// `JoinSet` returning an [`AbortHandle`] that can be used to remotely + /// cancel the task. + /// + /// The provided future will start running in the background immediately + /// when this method is called, even if you don't await anything on this + /// `JoinSet`. + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_on<F>(&mut self, task: F, handle: &Handle) -> AbortHandle + where + F: Future<Output = T>, + F: Send + 'static, + T: Send, + { + self.insert(handle.spawn(task)) + } + + /// Spawn the provided task on the current [`LocalSet`] and store it in this + /// `JoinSet`, returning an [`AbortHandle`] that can be used to remotely + /// cancel the task. + /// + /// The provided future will start running in the background immediately + /// when this method is called, even if you don't await anything on this + /// `JoinSet`. + /// + /// # Panics + /// + /// This method panics if it is called outside of a `LocalSet`. + /// + /// [`LocalSet`]: crate::task::LocalSet + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_local<F>(&mut self, task: F) -> AbortHandle + where + F: Future<Output = T>, + F: 'static, + { + self.insert(crate::task::spawn_local(task)) + } + + /// Spawn the provided task on the provided [`LocalSet`] and store it in + /// this `JoinSet`, returning an [`AbortHandle`] that can be used to + /// remotely cancel the task. + /// + /// Unlike the [`spawn_local`] method, this method may be used to spawn local + /// tasks on a `LocalSet` that is _not_ currently running. The provided + /// future will start running whenever the `LocalSet` is next started. + /// + /// [`LocalSet`]: crate::task::LocalSet + /// [`AbortHandle`]: crate::task::AbortHandle + /// [`spawn_local`]: Self::spawn_local + #[track_caller] + pub fn spawn_local_on<F>(&mut self, task: F, local_set: &LocalSet) -> AbortHandle + where + F: Future<Output = T>, + F: 'static, + { + self.insert(local_set.spawn_local(task)) + } + + fn insert(&mut self, jh: JoinHandle<T>) -> AbortHandle { + let abort = jh.abort_handle(); + let mut entry = self.inner.insert_idle(jh); + + // Set the waker that is notified when the task completes. + entry.with_value_and_context(|jh, ctx| jh.set_join_waker(ctx.waker())); + abort + } + + /// Waits until one of the tasks in the set completes and returns its output. + /// + /// Returns `None` if the set is empty. + /// + /// # Cancel Safety + /// + /// This method is cancel safe. If `join_next` is used as the event in a `tokio::select!` + /// statement and some other branch completes first, it is guaranteed that no tasks were + /// removed from this `JoinSet`. + pub async fn join_next(&mut self) -> Option<Result<T, JoinError>> { + crate::future::poll_fn(|cx| self.poll_join_next(cx)).await + } + + /// Waits until one of the tasks in the set completes and returns its + /// output, along with the [task ID] of the completed task. + /// + /// Returns `None` if the set is empty. + /// + /// When this method returns an error, then the id of the task that failed can be accessed + /// using the [`JoinError::id`] method. + /// + /// # Cancel Safety + /// + /// This method is cancel safe. If `join_next_with_id` is used as the event in a `tokio::select!` + /// statement and some other branch completes first, it is guaranteed that no tasks were + /// removed from this `JoinSet`. + /// + /// [task ID]: crate::task::Id + /// [`JoinError::id`]: fn@crate::task::JoinError::id + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub async fn join_next_with_id(&mut self) -> Option<Result<(Id, T), JoinError>> { + crate::future::poll_fn(|cx| self.poll_join_next_with_id(cx)).await + } + + /// Aborts all tasks and waits for them to finish shutting down. + /// + /// Calling this method is equivalent to calling [`abort_all`] and then calling [`join_next`] in + /// a loop until it returns `None`. + /// + /// This method ignores any panics in the tasks shutting down. When this call returns, the + /// `JoinSet` will be empty. + /// + /// [`abort_all`]: fn@Self::abort_all + /// [`join_next`]: fn@Self::join_next + pub async fn shutdown(&mut self) { + self.abort_all(); + while self.join_next().await.is_some() {} + } + + /// Aborts all tasks on this `JoinSet`. + /// + /// This does not remove the tasks from the `JoinSet`. To wait for the tasks to complete + /// cancellation, you should call `join_next` in a loop until the `JoinSet` is empty. + pub fn abort_all(&mut self) { + self.inner.for_each(|jh| jh.abort()); + } + + /// Removes all tasks from this `JoinSet` without aborting them. + /// + /// The tasks removed by this call will continue to run in the background even if the `JoinSet` + /// is dropped. + pub fn detach_all(&mut self) { + self.inner.drain(drop); + } + + /// Polls for one of the tasks in the set to complete. + /// + /// If this returns `Poll::Ready(Some(_))`, then the task that completed is removed from the set. + /// + /// When the method returns `Poll::Pending`, the `Waker` in the provided `Context` is scheduled + /// to receive a wakeup when a task in the `JoinSet` completes. Note that on multiple calls to + /// `poll_join_next`, only the `Waker` from the `Context` passed to the most recent call is + /// scheduled to receive a wakeup. + /// + /// # Returns + /// + /// This function returns: + /// + /// * `Poll::Pending` if the `JoinSet` is not empty but there is no task whose output is + /// available right now. + /// * `Poll::Ready(Some(Ok(value)))` if one of the tasks in this `JoinSet` has completed. + /// The `value` is the return value of one of the tasks that completed. + /// * `Poll::Ready(Some(Err(err)))` if one of the tasks in this `JoinSet` has panicked or been + /// aborted. The `err` is the `JoinError` from the panicked/aborted task. + /// * `Poll::Ready(None)` if the `JoinSet` is empty. + /// + /// Note that this method may return `Poll::Pending` even if one of the tasks has completed. + /// This can happen if the [coop budget] is reached. + /// + /// [coop budget]: crate::task#cooperative-scheduling + fn poll_join_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<T, JoinError>>> { + // The call to `pop_notified` moves the entry to the `idle` list. It is moved back to + // the `notified` list if the waker is notified in the `poll` call below. + let mut entry = match self.inner.pop_notified(cx.waker()) { + Some(entry) => entry, + None => { + if self.is_empty() { + return Poll::Ready(None); + } else { + // The waker was set by `pop_notified`. + return Poll::Pending; + } + } + }; + + let res = entry.with_value_and_context(|jh, ctx| Pin::new(jh).poll(ctx)); + + if let Poll::Ready(res) = res { + let _entry = entry.remove(); + Poll::Ready(Some(res)) + } else { + // A JoinHandle generally won't emit a wakeup without being ready unless + // the coop limit has been reached. We yield to the executor in this + // case. + cx.waker().wake_by_ref(); + Poll::Pending + } + } + + /// Polls for one of the tasks in the set to complete. + /// + /// If this returns `Poll::Ready(Some(_))`, then the task that completed is removed from the set. + /// + /// When the method returns `Poll::Pending`, the `Waker` in the provided `Context` is scheduled + /// to receive a wakeup when a task in the `JoinSet` completes. Note that on multiple calls to + /// `poll_join_next`, only the `Waker` from the `Context` passed to the most recent call is + /// scheduled to receive a wakeup. + /// + /// # Returns + /// + /// This function returns: + /// + /// * `Poll::Pending` if the `JoinSet` is not empty but there is no task whose output is + /// available right now. + /// * `Poll::Ready(Some(Ok((id, value))))` if one of the tasks in this `JoinSet` has completed. + /// The `value` is the return value of one of the tasks that completed, and + /// `id` is the [task ID] of that task. + /// * `Poll::Ready(Some(Err(err)))` if one of the tasks in this `JoinSet` has panicked or been + /// aborted. The `err` is the `JoinError` from the panicked/aborted task. + /// * `Poll::Ready(None)` if the `JoinSet` is empty. + /// + /// Note that this method may return `Poll::Pending` even if one of the tasks has completed. + /// This can happen if the [coop budget] is reached. + /// + /// [coop budget]: crate::task#cooperative-scheduling + /// [task ID]: crate::task::Id + #[cfg(tokio_unstable)] + fn poll_join_next_with_id( + &mut self, + cx: &mut Context<'_>, + ) -> Poll<Option<Result<(Id, T), JoinError>>> { + // The call to `pop_notified` moves the entry to the `idle` list. It is moved back to + // the `notified` list if the waker is notified in the `poll` call below. + let mut entry = match self.inner.pop_notified(cx.waker()) { + Some(entry) => entry, + None => { + if self.is_empty() { + return Poll::Ready(None); + } else { + // The waker was set by `pop_notified`. + return Poll::Pending; + } + } + }; + + let res = entry.with_value_and_context(|jh, ctx| Pin::new(jh).poll(ctx)); + + if let Poll::Ready(res) = res { + let entry = entry.remove(); + // If the task succeeded, add the task ID to the output. Otherwise, the + // `JoinError` will already have the task's ID. + Poll::Ready(Some(res.map(|output| (entry.id(), output)))) + } else { + // A JoinHandle generally won't emit a wakeup without being ready unless + // the coop limit has been reached. We yield to the executor in this + // case. + cx.waker().wake_by_ref(); + Poll::Pending + } + } +} + +impl<T> Drop for JoinSet<T> { + fn drop(&mut self) { + self.inner.drain(|join_handle| join_handle.abort()); + } +} + +impl<T> fmt::Debug for JoinSet<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("JoinSet").field("len", &self.len()).finish() + } +} + +impl<T> Default for JoinSet<T> { + fn default() -> Self { + Self::new() + } +} + +// === impl Builder === + +#[cfg(all(tokio_unstable, feature = "tracing"))] +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] +impl<'a, T: 'static> Builder<'a, T> { + /// Assigns a name to the task which will be spawned. + pub fn name(self, name: &'a str) -> Self { + let builder = self.builder.name(name); + Self { builder, ..self } + } + + /// Spawn the provided task with this builder's settings and store it in the + /// [`JoinSet`], returning an [`AbortHandle`] that can be used to remotely + /// cancel the task. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn<F>(self, future: F) -> std::io::Result<AbortHandle> + where + F: Future<Output = T>, + F: Send + 'static, + T: Send, + { + Ok(self.joinset.insert(self.builder.spawn(future)?)) + } + + /// Spawn the provided task on the provided [runtime handle] with this + /// builder's settings, and store it in the [`JoinSet`]. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// + /// [`AbortHandle`]: crate::task::AbortHandle + /// [runtime handle]: crate::runtime::Handle + #[track_caller] + pub fn spawn_on<F>(self, future: F, handle: &Handle) -> std::io::Result<AbortHandle> + where + F: Future<Output = T>, + F: Send + 'static, + T: Send, + { + Ok(self.joinset.insert(self.builder.spawn_on(future, handle)?)) + } + + /// Spawn the provided task on the current [`LocalSet`] with this builder's + /// settings, and store it in the [`JoinSet`]. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// # Panics + /// + /// This method panics if it is called outside of a `LocalSet`. + /// + /// [`LocalSet`]: crate::task::LocalSet + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_local<F>(self, future: F) -> std::io::Result<AbortHandle> + where + F: Future<Output = T>, + F: 'static, + { + Ok(self.joinset.insert(self.builder.spawn_local(future)?)) + } + + /// Spawn the provided task on the provided [`LocalSet`] with this builder's + /// settings, and store it in the [`JoinSet`]. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// [`LocalSet`]: crate::task::LocalSet + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_local_on<F>(self, future: F, local_set: &LocalSet) -> std::io::Result<AbortHandle> + where + F: Future<Output = T>, + F: 'static, + { + Ok(self + .joinset + .insert(self.builder.spawn_local_on(future, local_set)?)) + } +} + +// Manual `Debug` impl so that `Builder` is `Debug` regardless of whether `T` is +// `Debug`. +#[cfg(all(tokio_unstable, feature = "tracing"))] +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] +impl<'a, T> fmt::Debug for Builder<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("join_set::Builder") + .field("joinset", &self.joinset) + .field("builder", &self.builder) + .finish() + } +} diff --git a/src/task/local.rs b/src/task/local.rs index 4a5d313..0675faa 100644 --- a/src/task/local.rs +++ b/src/task/local.rs @@ -1,8 +1,10 @@ //! Runs `!Send` futures on the current thread. +use crate::loom::cell::UnsafeCell; use crate::loom::sync::{Arc, Mutex}; use crate::runtime::task::{self, JoinHandle, LocalOwnedTasks, Task}; +use crate::runtime::{context, ThreadId}; use crate::sync::AtomicWaker; -use crate::util::VecDequeCell; +use crate::util::RcCell; use std::cell::Cell; use std::collections::VecDeque; @@ -10,6 +12,7 @@ use std::fmt; use std::future::Future; use std::marker::PhantomData; use std::pin::Pin; +use std::rc::Rc; use std::task::Poll; use pin_project_lite::pin_project; @@ -159,7 +162,7 @@ cfg_rt! { /// tokio::task::spawn_local(run_task(new_task)); /// } /// // If the while loop returns, then all the LocalSpawner - /// // objects have have been dropped. + /// // objects have been dropped. /// }); /// /// // This will return once all senders are dropped and all @@ -215,7 +218,7 @@ cfg_rt! { tick: Cell<u8>, /// State available from thread-local. - context: Context, + context: Rc<Context>, /// This type should not be Send. _not_send: PhantomData<*const ()>, @@ -224,23 +227,44 @@ cfg_rt! { /// State available from the thread-local. struct Context { - /// Collection of all active tasks spawned onto this executor. - owned: LocalOwnedTasks<Arc<Shared>>, - - /// Local run queue sender and receiver. - queue: VecDequeCell<task::Notified<Arc<Shared>>>, - /// State shared between threads. shared: Arc<Shared>, + + /// True if a task panicked without being handled and the local set is + /// configured to shutdown on unhandled panic. + unhandled_panic: Cell<bool>, } /// LocalSet state shared between threads. struct Shared { + /// # Safety + /// + /// This field must *only* be accessed from the thread that owns the + /// `LocalSet` (i.e., `Thread::current().id() == owner`). + local_state: LocalState, + /// Remote run queue sender. queue: Mutex<Option<VecDeque<task::Notified<Arc<Shared>>>>>, /// Wake the `LocalSet` task. waker: AtomicWaker, + + /// How to respond to unhandled task panics. + #[cfg(tokio_unstable)] + pub(crate) unhandled_panic: crate::runtime::UnhandledPanic, +} + +/// Tracks the `LocalSet` state that must only be accessed from the thread that +/// created the `LocalSet`. +struct LocalState { + /// The `ThreadId` of the thread that owns the `LocalSet`. + owner: ThreadId, + + /// Local run queue sender and receiver. + local_queue: UnsafeCell<VecDeque<task::Notified<Arc<Shared>>>>, + + /// Collection of all active tasks spawned onto this executor. + owned: LocalOwnedTasks<Arc<Shared>>, } pin_project! { @@ -252,17 +276,30 @@ pin_project! { } } -scoped_thread_local!(static CURRENT: Context); +tokio_thread_local!(static CURRENT: LocalData = const { LocalData { + ctx: RcCell::new(), +} }); + +struct LocalData { + ctx: RcCell<Context>, +} cfg_rt! { - /// Spawns a `!Send` future on the local task set. + /// Spawns a `!Send` future on the current [`LocalSet`]. /// - /// The spawned future will be run on the same thread that called `spawn_local.` - /// This may only be called from the context of a local task set. + /// The spawned future will run on the same thread that called `spawn_local`. + /// + /// The provided future will start running in the background immediately + /// when `spawn_local` is called, even if you don't await the returned + /// `JoinHandle`. /// /// # Panics /// - /// - This function panics if called outside of a local task set. + /// This function panics if called outside of a [`LocalSet`]. + /// + /// Note that if [`tokio::spawn`] is used from within a `LocalSet`, the + /// resulting new task will _not_ be inside the `LocalSet`, so you must use + /// use `spawn_local` if you want to stay within the `LocalSet`. /// /// # Examples /// @@ -286,7 +323,10 @@ cfg_rt! { /// }).await; /// } /// ``` - #[cfg_attr(tokio_track_caller, track_caller)] + /// + /// [`LocalSet`]: struct@crate::task::LocalSet + /// [`tokio::spawn`]: fn@crate::task::spawn + #[track_caller] pub fn spawn_local<F>(future: F) -> JoinHandle<F::Output> where F: Future + 'static, @@ -295,23 +335,16 @@ cfg_rt! { spawn_local_inner(future, None) } + + #[track_caller] pub(super) fn spawn_local_inner<F>(future: F, name: Option<&str>) -> JoinHandle<F::Output> where F: Future + 'static, F::Output: 'static { - let future = crate::util::trace::task(future, "local", name); - CURRENT.with(|maybe_cx| { - let cx = maybe_cx - .expect("`spawn_local` called from outside of a `task::LocalSet`"); - - let (handle, notified) = cx.owned.bind(future, cx.shared.clone()); - - if let Some(notified) = notified { - cx.shared.schedule(notified); - } - - handle - }) + match CURRENT.with(|LocalData { ctx, .. }| ctx.get()) { + None => panic!("`spawn_local` called from outside of a `task::LocalSet`"), + Some(cx) => cx.spawn(future, name) + } } } @@ -324,29 +357,72 @@ const MAX_TASKS_PER_TICK: usize = 61; /// How often it check the remote queue first. const REMOTE_FIRST_INTERVAL: u8 = 31; +/// Context guard for LocalSet +pub struct LocalEnterGuard(Option<Rc<Context>>); + +impl Drop for LocalEnterGuard { + fn drop(&mut self) { + CURRENT.with(|LocalData { ctx, .. }| { + ctx.set(self.0.take()); + }) + } +} + +impl fmt::Debug for LocalEnterGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalEnterGuard").finish() + } +} + impl LocalSet { /// Returns a new local task set. pub fn new() -> LocalSet { + let owner = context::thread_id().expect("cannot create LocalSet during thread shutdown"); + LocalSet { tick: Cell::new(0), - context: Context { - owned: LocalOwnedTasks::new(), - queue: VecDequeCell::with_capacity(INITIAL_CAPACITY), + context: Rc::new(Context { shared: Arc::new(Shared { + local_state: LocalState { + owner, + owned: LocalOwnedTasks::new(), + local_queue: UnsafeCell::new(VecDeque::with_capacity(INITIAL_CAPACITY)), + }, queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))), waker: AtomicWaker::new(), + #[cfg(tokio_unstable)] + unhandled_panic: crate::runtime::UnhandledPanic::Ignore, }), - }, + unhandled_panic: Cell::new(false), + }), _not_send: PhantomData, } } + /// Enters the context of this `LocalSet`. + /// + /// The [`spawn_local`] method will spawn tasks on the `LocalSet` whose + /// context you are inside. + /// + /// [`spawn_local`]: fn@crate::task::spawn_local + pub fn enter(&self) -> LocalEnterGuard { + CURRENT.with(|LocalData { ctx, .. }| { + let old = ctx.replace(Some(self.context.clone())); + LocalEnterGuard(old) + }) + } + /// Spawns a `!Send` task onto the local task set. /// /// This task is guaranteed to be run on the current thread. /// /// Unlike the free function [`spawn_local`], this method may be used to - /// spawn local tasks when the task set is _not_ running. For example: + /// spawn local tasks when the `LocalSet` is _not_ running. The provided + /// future will start running once the `LocalSet` is next started, even if + /// you don't await the returned `JoinHandle`. + /// + /// # Examples + /// /// ```rust /// use tokio::task; /// @@ -377,22 +453,13 @@ impl LocalSet { /// } /// ``` /// [`spawn_local`]: fn@spawn_local - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub fn spawn_local<F>(&self, future: F) -> JoinHandle<F::Output> where F: Future + 'static, F::Output: 'static, { - let future = crate::util::trace::task(future, "local", None); - - let (handle, notified) = self.context.owned.bind(future, self.context.shared.clone()); - - if let Some(notified) = notified { - self.context.shared.schedule(notified); - } - - self.context.shared.waker.wake(); - handle + self.spawn_named(future, None) } /// Runs a future to completion on the provided runtime, driving any local @@ -457,6 +524,7 @@ impl LocalSet { /// [`Runtime::block_on`]: method@crate::runtime::Runtime::block_on /// [in-place blocking]: fn@crate::task::block_in_place /// [`spawn_blocking`]: fn@crate::task::spawn_blocking + #[track_caller] #[cfg(feature = "rt")] #[cfg_attr(docsrs, doc(cfg(feature = "rt")))] pub fn block_on<F>(&self, rt: &crate::runtime::Runtime, future: F) -> F::Output @@ -505,10 +573,36 @@ impl LocalSet { run_until.await } + pub(in crate::task) fn spawn_named<F>( + &self, + future: F, + name: Option<&str>, + ) -> JoinHandle<F::Output> + where + F: Future + 'static, + F::Output: 'static, + { + let handle = self.context.spawn(future, name); + + // Because a task was spawned from *outside* the `LocalSet`, wake the + // `LocalSet` future to execute the new task, if it hasn't been woken. + // + // Spawning via the free fn `spawn` does not require this, as it can + // only be called from *within* a future executing on the `LocalSet` — + // in that case, the `LocalSet` must already be awake. + self.context.shared.waker.wake(); + handle + } + /// Ticks the scheduler, returning whether the local future needs to be /// notified again. fn tick(&self) -> bool { for _ in 0..MAX_TASKS_PER_TICK { + // Make sure we didn't hit an unhandled panic + if self.context.unhandled_panic.get() { + panic!("a spawned task panicked and the LocalSet is configured to shutdown on unhandled panic"); + } + match self.next_task() { // Run the task // @@ -518,7 +612,7 @@ impl LocalSet { // task initially. Because `LocalSet` itself is `!Send`, and // `spawn_local` spawns into the `LocalSet` on the current // thread, the invariant is maintained. - Some(task) => crate::coop::budget(|| task.run()), + Some(task) => crate::runtime::coop::budget(|| task.run()), // We have fully drained the queue of notified tasks, so the // local future doesn't need to be notified again — it can wait // until something else wakes a task in the local set. @@ -540,9 +634,9 @@ impl LocalSet { .lock() .as_mut() .and_then(|queue| queue.pop_front()) - .or_else(|| self.context.queue.pop_front()) + .or_else(|| self.pop_local()) } else { - self.context.queue.pop_front().or_else(|| { + self.pop_local().or_else(|| { self.context .shared .queue @@ -552,11 +646,145 @@ impl LocalSet { }) }; - task.map(|task| self.context.owned.assert_owner(task)) + task.map(|task| unsafe { + // Safety: because the `LocalSet` itself is `!Send`, we know we are + // on the same thread if we have access to the `LocalSet`, and can + // therefore access the local run queue. + self.context.shared.local_state.assert_owner(task) + }) + } + + fn pop_local(&self) -> Option<task::Notified<Arc<Shared>>> { + unsafe { + // Safety: because the `LocalSet` itself is `!Send`, we know we are + // on the same thread if we have access to the `LocalSet`, and can + // therefore access the local run queue. + self.context.shared.local_state.task_pop_front() + } } fn with<T>(&self, f: impl FnOnce() -> T) -> T { - CURRENT.set(&self.context, f) + CURRENT.with(|LocalData { ctx, .. }| { + struct Reset<'a> { + ctx_ref: &'a RcCell<Context>, + val: Option<Rc<Context>>, + } + impl<'a> Drop for Reset<'a> { + fn drop(&mut self) { + self.ctx_ref.set(self.val.take()); + } + } + let old = ctx.replace(Some(self.context.clone())); + + let _reset = Reset { + ctx_ref: ctx, + val: old, + }; + + f() + }) + } + + /// This method is like `with`, but it just calls `f` without setting the thread-local if that + /// fails. + fn with_if_possible<T>(&self, f: impl FnOnce() -> T) -> T { + let mut f = Some(f); + + let res = CURRENT.try_with(|LocalData { ctx, .. }| { + struct Reset<'a> { + ctx_ref: &'a RcCell<Context>, + val: Option<Rc<Context>>, + } + impl<'a> Drop for Reset<'a> { + fn drop(&mut self) { + self.ctx_ref.replace(self.val.take()); + } + } + let old = ctx.replace(Some(self.context.clone())); + + let _reset = Reset { + ctx_ref: ctx, + val: old, + }; + + (f.take().unwrap())() + }); + + match res { + Ok(res) => res, + Err(_access_error) => (f.take().unwrap())(), + } + } +} + +cfg_unstable! { + impl LocalSet { + /// Configure how the `LocalSet` responds to an unhandled panic on a + /// spawned task. + /// + /// By default, an unhandled panic (i.e. a panic not caught by + /// [`std::panic::catch_unwind`]) has no impact on the `LocalSet`'s + /// execution. The panic is error value is forwarded to the task's + /// [`JoinHandle`] and all other spawned tasks continue running. + /// + /// The `unhandled_panic` option enables configuring this behavior. + /// + /// * `UnhandledPanic::Ignore` is the default behavior. Panics on + /// spawned tasks have no impact on the `LocalSet`'s execution. + /// * `UnhandledPanic::ShutdownRuntime` will force the `LocalSet` to + /// shutdown immediately when a spawned task panics even if that + /// task's `JoinHandle` has not been dropped. All other spawned tasks + /// will immediately terminate and further calls to + /// [`LocalSet::block_on`] and [`LocalSet::run_until`] will panic. + /// + /// # Panics + /// + /// This method panics if called after the `LocalSet` has started + /// running. + /// + /// # Unstable + /// + /// This option is currently unstable and its implementation is + /// incomplete. The API may change or be removed in the future. See + /// tokio-rs/tokio#4516 for more details. + /// + /// # Examples + /// + /// The following demonstrates a `LocalSet` configured to shutdown on + /// panic. The first spawned task panics and results in the `LocalSet` + /// shutting down. The second spawned task never has a chance to + /// execute. The call to `run_until` will panic due to the runtime being + /// forcibly shutdown. + /// + /// ```should_panic + /// use tokio::runtime::UnhandledPanic; + /// + /// # #[tokio::main] + /// # async fn main() { + /// tokio::task::LocalSet::new() + /// .unhandled_panic(UnhandledPanic::ShutdownRuntime) + /// .run_until(async { + /// tokio::task::spawn_local(async { panic!("boom"); }); + /// tokio::task::spawn_local(async { + /// // This task never completes + /// }); + /// + /// // Do some work, but `run_until` will panic before it completes + /// # loop { tokio::task::yield_now().await; } + /// }) + /// .await; + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + pub fn unhandled_panic(&mut self, behavior: crate::runtime::UnhandledPanic) -> &mut Self { + // TODO: This should be set as a builder + Rc::get_mut(&mut self.context) + .and_then(|ctx| Arc::get_mut(&mut ctx.shared)) + .expect("Unhandled Panic behavior modified after starting LocalSet") + .unhandled_panic = behavior; + self + } } } @@ -578,7 +806,10 @@ impl Future for LocalSet { // there are still tasks remaining in the run queue. cx.waker().wake_by_ref(); Poll::Pending - } else if self.context.owned.is_empty() { + + // Safety: called from the thread that owns `LocalSet`. Because + // `LocalSet` is `!Send`, this is safe. + } else if unsafe { self.context.shared.local_state.owned_is_empty() } { // If the scheduler has no remaining futures, we're done! Poll::Ready(()) } else { @@ -598,14 +829,34 @@ impl Default for LocalSet { impl Drop for LocalSet { fn drop(&mut self) { - self.with(|| { + self.with_if_possible(|| { // Shut down all tasks in the LocalOwnedTasks and close it to // prevent new tasks from ever being added. - self.context.owned.close_and_shutdown_all(); + unsafe { + // Safety: called from the thread that owns `LocalSet` + self.context.shared.local_state.close_and_shutdown_all(); + } // We already called shutdown on all tasks above, so there is no // need to call shutdown. - for task in self.context.queue.take() { + + // Safety: note that this *intentionally* bypasses the unsafe + // `Shared::local_queue()` method. This is in order to avoid the + // debug assertion that we are on the thread that owns the + // `LocalSet`, because on some systems (e.g. at least some macOS + // versions), attempting to get the current thread ID can panic due + // to the thread's local data that stores the thread ID being + // dropped *before* the `LocalSet`. + // + // Despite avoiding the assertion here, it is safe for us to access + // the local queue in `Drop`, because the `LocalSet` itself is + // `!Send`, so we can reasonably guarantee that it will not be + // `Drop`ped from another thread. + let local_queue = unsafe { + // Safety: called from the thread that owns `LocalSet` + self.context.shared.local_state.take_local_queue() + }; + for task in local_queue { drop(task); } @@ -616,11 +867,41 @@ impl Drop for LocalSet { drop(task); } - assert!(self.context.owned.is_empty()); + // Safety: called from the thread that owns `LocalSet` + assert!(unsafe { self.context.shared.local_state.owned_is_empty() }); }); } } +// === impl Context === + +impl Context { + #[track_caller] + fn spawn<F>(&self, future: F, name: Option<&str>) -> JoinHandle<F::Output> + where + F: Future + 'static, + F::Output: 'static, + { + let id = crate::runtime::task::Id::next(); + let future = crate::util::trace::task(future, "local", name, id.as_u64()); + + // Safety: called from the thread that owns the `LocalSet` + let (handle, notified) = { + self.shared.local_state.assert_called_from_owner_thread(); + self.shared + .local_state + .owned + .bind(future, self.shared.clone(), id) + }; + + if let Some(notified) = notified { + self.shared.schedule(notified); + } + + handle + } +} + // === impl LocalFuture === impl<T: Future> Future for RunUntil<'_, T> { @@ -636,10 +917,10 @@ impl<T: Future> Future for RunUntil<'_, T> { .waker .register_by_ref(cx.waker()); - let _no_blocking = crate::runtime::enter::disallow_blocking(); + let _no_blocking = crate::runtime::context::disallow_block_in_place(); let f = me.future; - if let Poll::Ready(output) = crate::coop::budget(|| f.poll(cx)) { + if let Poll::Ready(output) = f.poll(cx) { return Poll::Ready(output); } @@ -657,21 +938,41 @@ impl<T: Future> Future for RunUntil<'_, T> { impl Shared { /// Schedule the provided task on the scheduler. fn schedule(&self, task: task::Notified<Arc<Self>>) { - CURRENT.with(|maybe_cx| match maybe_cx { - Some(cx) if cx.shared.ptr_eq(self) => { - cx.queue.push_back(task); - } - _ => { - // First check whether the queue is still there (if not, the - // LocalSet is dropped). Then push to it if so, and if not, - // do nothing. - let mut lock = self.queue.lock(); - - if let Some(queue) = lock.as_mut() { - queue.push_back(task); - drop(lock); + CURRENT.with(|localdata| { + match localdata.ctx.get() { + Some(cx) if cx.shared.ptr_eq(self) => unsafe { + // Safety: if the current `LocalSet` context points to this + // `LocalSet`, then we are on the thread that owns it. + cx.shared.local_state.task_push_back(task); + }, + + // We are on the thread that owns the `LocalSet`, so we can + // wake to the local queue. + _ if context::thread_id().ok() == Some(self.local_state.owner) => { + unsafe { + // Safety: we just checked that the thread ID matches + // the localset's owner, so this is safe. + self.local_state.task_push_back(task); + } + // We still have to wake the `LocalSet`, because it isn't + // currently being polled. self.waker.wake(); } + + // We are *not* on the thread that owns the `LocalSet`, so we + // have to wake to the remote queue. + _ => { + // First, check whether the queue is still there (if not, the + // LocalSet is dropped). Then push to it if so, and if not, + // do nothing. + let mut lock = self.queue.lock(); + + if let Some(queue) = lock.as_mut() { + queue.push_back(task); + drop(lock); + self.waker.wake(); + } + } } }); } @@ -681,16 +982,194 @@ impl Shared { } } +// This is safe because (and only because) we *pinky pwomise* to never touch the +// local run queue except from the thread that owns the `LocalSet`. +unsafe impl Sync for Shared {} + impl task::Schedule for Arc<Shared> { fn release(&self, task: &Task<Self>) -> Option<Task<Self>> { - CURRENT.with(|maybe_cx| { - let cx = maybe_cx.expect("scheduler context missing"); - assert!(cx.shared.ptr_eq(self)); - cx.owned.remove(task) - }) + // Safety, this is always called from the thread that owns `LocalSet` + unsafe { self.local_state.task_remove(task) } } fn schedule(&self, task: task::Notified<Self>) { Shared::schedule(self, task); } + + cfg_unstable! { + fn unhandled_panic(&self) { + use crate::runtime::UnhandledPanic; + + match self.unhandled_panic { + UnhandledPanic::Ignore => { + // Do nothing + } + UnhandledPanic::ShutdownRuntime => { + // This hook is only called from within the runtime, so + // `CURRENT` should match with `&self`, i.e. there is no + // opportunity for a nested scheduler to be called. + CURRENT.with(|LocalData { ctx, .. }| match ctx.get() { + Some(cx) if Arc::ptr_eq(self, &cx.shared) => { + cx.unhandled_panic.set(true); + // Safety: this is always called from the thread that owns `LocalSet` + unsafe { cx.shared.local_state.close_and_shutdown_all(); } + } + _ => unreachable!("runtime core not set in CURRENT thread-local"), + }) + } + } + } + } +} + +impl LocalState { + unsafe fn task_pop_front(&self) -> Option<task::Notified<Arc<Shared>>> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.local_queue.with_mut(|ptr| (*ptr).pop_front()) + } + + unsafe fn task_push_back(&self, task: task::Notified<Arc<Shared>>) { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.local_queue.with_mut(|ptr| (*ptr).push_back(task)) + } + + unsafe fn take_local_queue(&self) -> VecDeque<task::Notified<Arc<Shared>>> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.local_queue.with_mut(|ptr| std::mem::take(&mut (*ptr))) + } + + unsafe fn task_remove(&self, task: &Task<Arc<Shared>>) -> Option<Task<Arc<Shared>>> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.remove(task) + } + + /// Returns true if the `LocalSet` does not have any spawned tasks + unsafe fn owned_is_empty(&self) -> bool { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.is_empty() + } + + unsafe fn assert_owner( + &self, + task: task::Notified<Arc<Shared>>, + ) -> task::LocalNotified<Arc<Shared>> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.assert_owner(task) + } + + unsafe fn close_and_shutdown_all(&self) { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.close_and_shutdown_all() + } + + #[track_caller] + fn assert_called_from_owner_thread(&self) { + // FreeBSD has some weirdness around thread-local destruction. + // TODO: remove this hack when thread id is cleaned up + #[cfg(not(any(target_os = "openbsd", target_os = "freebsd")))] + debug_assert!( + // if we couldn't get the thread ID because we're dropping the local + // data, skip the assertion --- the `Drop` impl is not going to be + // called from another thread, because `LocalSet` is `!Send` + context::thread_id() + .map(|id| id == self.owner) + .unwrap_or(true), + "`LocalSet`'s local run queue must not be accessed by another thread!" + ); + } +} + +// This is `Send` because it is stored in `Shared`. It is up to the caller to +// ensure they are on the same thread that owns the `LocalSet`. +unsafe impl Send for LocalState {} + +#[cfg(all(test, not(loom)))] +mod tests { + use super::*; + + // Does a `LocalSet` running on a current-thread runtime...basically work? + // + // This duplicates a test in `tests/task_local_set.rs`, but because this is + // a lib test, it wil run under Miri, so this is necessary to catch stacked + // borrows violations in the `LocalSet` implementation. + #[test] + fn local_current_thread_scheduler() { + let f = async { + LocalSet::new() + .run_until(async { + spawn_local(async {}).await.unwrap(); + }) + .await; + }; + crate::runtime::Builder::new_current_thread() + .build() + .expect("rt") + .block_on(f) + } + + // Tests that when a task on a `LocalSet` is woken by an io driver on the + // same thread, the task is woken to the localset's local queue rather than + // its remote queue. + // + // This test has to be defined in the `local.rs` file as a lib test, rather + // than in `tests/`, because it makes assertions about the local set's + // internal state. + #[test] + fn wakes_to_local_queue() { + use super::*; + use crate::sync::Notify; + let rt = crate::runtime::Builder::new_current_thread() + .build() + .expect("rt"); + rt.block_on(async { + let local = LocalSet::new(); + let notify = Arc::new(Notify::new()); + let task = local.spawn_local({ + let notify = notify.clone(); + async move { + notify.notified().await; + } + }); + let mut run_until = Box::pin(local.run_until(async move { + task.await.unwrap(); + })); + + // poll the run until future once + crate::future::poll_fn(|cx| { + let _ = run_until.as_mut().poll(cx); + Poll::Ready(()) + }) + .await; + + notify.notify_one(); + let task = unsafe { local.context.shared.local_state.task_pop_front() }; + // TODO(eliza): it would be nice to be able to assert that this is + // the local task. + assert!( + task.is_some(), + "task should have been notified to the LocalSet's local queue" + ); + }) + } } diff --git a/src/task/mod.rs b/src/task/mod.rs index ea98787..9b75370 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -243,7 +243,7 @@ //! //! #### unconstrained //! -//! If necessary, [`task::unconstrained`] lets you opt out a future of Tokio's cooperative +//! If necessary, [`task::unconstrained`] lets you opt a future out of of Tokio's cooperative //! scheduling. When a future is wrapped with `unconstrained`, it will never be forced to yield to //! Tokio. For example: //! @@ -278,8 +278,10 @@ cfg_rt! { pub use crate::runtime::task::{JoinError, JoinHandle}; - mod blocking; - pub use blocking::spawn_blocking; + cfg_not_wasi! { + mod blocking; + pub use blocking::spawn_blocking; + } mod spawn; pub use spawn::spawn; @@ -291,8 +293,13 @@ cfg_rt! { mod yield_now; pub use yield_now::yield_now; + cfg_unstable! { + mod consume_budget; + pub use consume_budget::consume_budget; + } + mod local; - pub use local::{spawn_local, LocalSet}; + pub use local::{spawn_local, LocalSet, LocalEnterGuard}; mod task_local; pub use task_local::LocalKey; @@ -300,6 +307,20 @@ cfg_rt! { mod unconstrained; pub use unconstrained::{unconstrained, Unconstrained}; + #[doc(inline)] + pub use join_set::JoinSet; + pub use crate::runtime::task::AbortHandle; + + // Uses #[cfg(...)] instead of macro since the macro adds docsrs annotations. + #[cfg(not(tokio_unstable))] + mod join_set; + #[cfg(tokio_unstable)] + pub mod join_set; + + cfg_unstable! { + pub use crate::runtime::task::{Id, id, try_id}; + } + cfg_trace! { mod builder; pub use builder::Builder; diff --git a/src/task/spawn.rs b/src/task/spawn.rs index 065d38f..66b0d67 100644 --- a/src/task/spawn.rs +++ b/src/task/spawn.rs @@ -1,4 +1,5 @@ -use crate::{task::JoinHandle, util::error::CONTEXT_MISSING_ERROR}; +use crate::runtime::Handle; +use crate::task::JoinHandle; use std::future::Future; @@ -6,6 +7,10 @@ cfg_rt! { /// Spawns a new asynchronous task, returning a /// [`JoinHandle`](super::JoinHandle) for it. /// + /// The provided future will start running in the background immediately + /// when `spawn` is called, even if you don't await the returned + /// `JoinHandle`. + /// /// Spawning a task enables the task to execute concurrently to other tasks. The /// spawned task may execute on the current thread, or it may be sent to a /// different thread to be executed. The specifics depend on the current @@ -49,6 +54,37 @@ cfg_rt! { /// } /// ``` /// + /// To run multiple tasks in parallel and receive their results, join + /// handles can be stored in a vector. + /// ``` + /// # #[tokio::main(flavor = "current_thread")] async fn main() { + /// async fn my_background_op(id: i32) -> String { + /// let s = format!("Starting background task {}.", id); + /// println!("{}", s); + /// s + /// } + /// + /// let ops = vec![1, 2, 3]; + /// let mut tasks = Vec::with_capacity(ops.len()); + /// for op in ops { + /// // This call will make them start running in the background + /// // immediately. + /// tasks.push(tokio::spawn(my_background_op(op))); + /// } + /// + /// let mut outputs = Vec::with_capacity(tasks.len()); + /// for task in tasks { + /// outputs.push(task.await.unwrap()); + /// } + /// println!("{:?}", outputs); + /// # } + /// ``` + /// This example pushes the tasks to `outputs` in the order they were + /// started in. If you do not care about the ordering of the outputs, then + /// you can also use a [`JoinSet`]. + /// + /// [`JoinSet`]: struct@crate::task::JoinSet + /// /// # Panics /// /// Panics if called from **outside** of the Tokio runtime. @@ -121,7 +157,7 @@ cfg_rt! { /// ```text /// error[E0391]: cycle detected when processing `main` /// ``` - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub fn spawn<T>(future: T) -> JoinHandle<T::Output> where T: Future + Send + 'static, @@ -136,14 +172,16 @@ cfg_rt! { } } - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub(super) fn spawn_inner<T>(future: T, name: Option<&str>) -> JoinHandle<T::Output> where T: Future + Send + 'static, T::Output: Send + 'static, { - let spawn_handle = crate::runtime::context::spawn_handle().expect(CONTEXT_MISSING_ERROR); - let task = crate::util::trace::task(future, "task", name); - spawn_handle.spawn(task) + use crate::runtime::task; + let id = task::Id::next(); + let task = crate::util::trace::task(future, "task", name, id.as_u64()); + let handle = Handle::current(); + handle.inner.spawn(task, id) } } diff --git a/src/task/task_local.rs b/src/task/task_local.rs index b6e7df4..d3b108f 100644 --- a/src/task/task_local.rs +++ b/src/task/task_local.rs @@ -1,11 +1,10 @@ -use pin_project_lite::pin_project; use std::cell::RefCell; use std::error::Error; use std::future::Future; use std::marker::PhantomPinned; use std::pin::Pin; use std::task::{Context, Poll}; -use std::{fmt, thread}; +use std::{fmt, mem, thread}; /// Declares a new task-local key of type [`tokio::task::LocalKey`]. /// @@ -48,9 +47,27 @@ macro_rules! task_local { } #[doc(hidden)] +#[cfg(not(tokio_no_const_thread_local))] #[macro_export] macro_rules! __task_local_inner { ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty) => { + $(#[$attr])* + $vis static $name: $crate::task::LocalKey<$t> = { + std::thread_local! { + static __KEY: std::cell::RefCell<Option<$t>> = const { std::cell::RefCell::new(None) }; + } + + $crate::task::LocalKey { inner: __KEY } + }; + }; +} + +#[doc(hidden)] +#[cfg(tokio_no_const_thread_local)] +#[macro_export] +macro_rules! __task_local_inner { + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty) => { + $(#[$attr])* $vis static $name: $crate::task::LocalKey<$t> = { std::thread_local! { static __KEY: std::cell::RefCell<Option<$t>> = std::cell::RefCell::new(None); @@ -63,7 +80,7 @@ macro_rules! __task_local_inner { /// A key for task-local data. /// -/// This type is generated by the `task_local!` macro. +/// This type is generated by the [`task_local!`] macro. /// /// Unlike [`std::thread::LocalKey`], `tokio::task::LocalKey` will /// _not_ lazily initialize the value on first access. Instead, the @@ -91,7 +108,9 @@ macro_rules! __task_local_inner { /// }).await; /// # } /// ``` +/// /// [`std::thread::LocalKey`]: struct@std::thread::LocalKey +/// [`task_local!`]: ../macro.task_local.html #[cfg_attr(docsrs, doc(cfg(feature = "rt")))] pub struct LocalKey<T: 'static> { #[doc(hidden)] @@ -103,6 +122,11 @@ impl<T: 'static> LocalKey<T> { /// /// On completion of `scope`, the task-local will be dropped. /// + /// ### Panics + /// + /// If you poll the returned future inside a call to [`with`] or + /// [`try_with`] on the same `LocalKey`, then the call to `poll` will panic. + /// /// ### Examples /// /// ``` @@ -116,6 +140,9 @@ impl<T: 'static> LocalKey<T> { /// }).await; /// # } /// ``` + /// + /// [`with`]: fn@Self::with + /// [`try_with`]: fn@Self::try_with pub fn scope<F>(&'static self, value: T, f: F) -> TaskLocalFuture<T, F> where F: Future, @@ -123,14 +150,19 @@ impl<T: 'static> LocalKey<T> { TaskLocalFuture { local: self, slot: Some(value), - future: f, + future: Some(f), _pinned: PhantomPinned, } } /// Sets a value `T` as the task-local value for the closure `F`. /// - /// On completion of `scope`, the task-local will be dropped. + /// On completion of `sync_scope`, the task-local will be dropped. + /// + /// ### Panics + /// + /// This method panics if called inside a call to [`with`] or [`try_with`] + /// on the same `LocalKey`. /// /// ### Examples /// @@ -145,34 +177,80 @@ impl<T: 'static> LocalKey<T> { /// }); /// # } /// ``` + /// + /// [`with`]: fn@Self::with + /// [`try_with`]: fn@Self::try_with + #[track_caller] pub fn sync_scope<F, R>(&'static self, value: T, f: F) -> R where F: FnOnce() -> R, { - let scope = TaskLocalFuture { - local: self, - slot: Some(value), - future: (), - _pinned: PhantomPinned, - }; - crate::pin!(scope); - scope.with_task(|_| f()) + let mut value = Some(value); + match self.scope_inner(&mut value, f) { + Ok(res) => res, + Err(err) => err.panic(), + } + } + + fn scope_inner<F, R>(&'static self, slot: &mut Option<T>, f: F) -> Result<R, ScopeInnerErr> + where + F: FnOnce() -> R, + { + struct Guard<'a, T: 'static> { + local: &'static LocalKey<T>, + slot: &'a mut Option<T>, + } + + impl<'a, T: 'static> Drop for Guard<'a, T> { + fn drop(&mut self) { + // This should not panic. + // + // We know that the RefCell was not borrowed before the call to + // `scope_inner`, so the only way for this to panic is if the + // closure has created but not destroyed a RefCell guard. + // However, we never give user-code access to the guards, so + // there's no way for user-code to forget to destroy a guard. + // + // The call to `with` also should not panic, since the + // thread-local wasn't destroyed when we first called + // `scope_inner`, and it shouldn't have gotten destroyed since + // then. + self.local.inner.with(|inner| { + let mut ref_mut = inner.borrow_mut(); + mem::swap(self.slot, &mut *ref_mut); + }); + } + } + + self.inner.try_with(|inner| { + inner + .try_borrow_mut() + .map(|mut ref_mut| mem::swap(slot, &mut *ref_mut)) + })??; + + let guard = Guard { local: self, slot }; + + let res = f(); + + drop(guard); + + Ok(res) } /// Accesses the current task-local and runs the provided closure. /// /// # Panics /// - /// This function will panic if not called within the context - /// of a future containing a task-local with the corresponding key. + /// This function will panic if the task local doesn't have a value set. + #[track_caller] pub fn with<F, R>(&'static self, f: F) -> R where F: FnOnce(&T) -> R, { - self.try_with(f).expect( - "cannot access a Task Local Storage value \ - without setting it via `LocalKey::set`", - ) + match self.try_with(f) { + Ok(res) => res, + Err(_) => panic!("cannot access a task-local storage value without setting it first"), + } } /// Accesses the current task-local and runs the provided closure. @@ -184,19 +262,32 @@ impl<T: 'static> LocalKey<T> { where F: FnOnce(&T) -> R, { - self.inner.with(|v| { - if let Some(val) = v.borrow().as_ref() { - Ok(f(val)) - } else { - Err(AccessError { _private: () }) - } - }) + // If called after the thread-local storing the task-local is destroyed, + // then we are outside of a closure where the task-local is set. + // + // Therefore, it is correct to return an AccessError if `try_with` + // returns an error. + let try_with_res = self.inner.try_with(|v| { + // This call to `borrow` cannot panic because no user-defined code + // runs while a `borrow_mut` call is active. + v.borrow().as_ref().map(f) + }); + + match try_with_res { + Ok(Some(res)) => Ok(res), + Ok(None) | Err(_) => Err(AccessError { _private: () }), + } } } impl<T: Copy + 'static> LocalKey<T> { /// Returns a copy of the task-local value /// if the task-local value implements `Copy`. + /// + /// # Panics + /// + /// This function will panic if the task local doesn't have a value set. + #[track_caller] pub fn get(&'static self) -> T { self.with(|v| *v) } @@ -208,76 +299,104 @@ impl<T: 'static> fmt::Debug for LocalKey<T> { } } -pin_project! { - /// A future that sets a value `T` of a task local for the future `F` during - /// its execution. - /// - /// The value of the task-local must be `'static` and will be dropped on the - /// completion of the future. - /// - /// Created by the function [`LocalKey::scope`](self::LocalKey::scope). - /// - /// ### Examples - /// - /// ``` - /// # async fn dox() { - /// tokio::task_local! { - /// static NUMBER: u32; - /// } - /// - /// NUMBER.scope(1, async move { - /// println!("task local value: {}", NUMBER.get()); - /// }).await; - /// # } - /// ``` - pub struct TaskLocalFuture<T, F> - where - T: 'static - { - local: &'static LocalKey<T>, - slot: Option<T>, - #[pin] - future: F, - #[pin] - _pinned: PhantomPinned, - } +/// A future that sets a value `T` of a task local for the future `F` during +/// its execution. +/// +/// The value of the task-local must be `'static` and will be dropped on the +/// completion of the future. +/// +/// Created by the function [`LocalKey::scope`](self::LocalKey::scope). +/// +/// ### Examples +/// +/// ``` +/// # async fn dox() { +/// tokio::task_local! { +/// static NUMBER: u32; +/// } +/// +/// NUMBER.scope(1, async move { +/// println!("task local value: {}", NUMBER.get()); +/// }).await; +/// # } +/// ``` +// Doesn't use pin_project due to custom Drop. +pub struct TaskLocalFuture<T, F> +where + T: 'static, +{ + local: &'static LocalKey<T>, + slot: Option<T>, + future: Option<F>, + _pinned: PhantomPinned, } -impl<T: 'static, F> TaskLocalFuture<T, F> { - fn with_task<F2: FnOnce(Pin<&mut F>) -> R, R>(self: Pin<&mut Self>, f: F2) -> R { - struct Guard<'a, T: 'static> { - local: &'static LocalKey<T>, - slot: &'a mut Option<T>, - prev: Option<T>, - } - - impl<T> Drop for Guard<'_, T> { - fn drop(&mut self) { - let value = self.local.inner.with(|c| c.replace(self.prev.take())); - *self.slot = value; - } - } +impl<T: 'static, F: Future> Future for TaskLocalFuture<T, F> { + type Output = F::Output; - let mut project = self.project(); - let val = project.slot.take(); + #[track_caller] + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + // safety: The TaskLocalFuture struct is `!Unpin` so there is no way to + // move `self.future` from now on. + let this = unsafe { Pin::into_inner_unchecked(self) }; + let mut future_opt = unsafe { Pin::new_unchecked(&mut this.future) }; - let prev = project.local.inner.with(|c| c.replace(val)); + let res = + this.local + .scope_inner(&mut this.slot, || match future_opt.as_mut().as_pin_mut() { + Some(fut) => { + let res = fut.poll(cx); + if res.is_ready() { + future_opt.set(None); + } + Some(res) + } + None => None, + }); - let _guard = Guard { - prev, - slot: &mut project.slot, - local: *project.local, - }; + match res { + Ok(Some(res)) => res, + Ok(None) => panic!("`TaskLocalFuture` polled after completion"), + Err(err) => err.panic(), + } + } +} - f(project.future) +impl<T: 'static, F> Drop for TaskLocalFuture<T, F> { + fn drop(&mut self) { + if mem::needs_drop::<F>() && self.future.is_some() { + // Drop the future while the task-local is set, if possible. Otherwise + // the future is dropped normally when the `Option<F>` field drops. + let future = &mut self.future; + let _ = self.local.scope_inner(&mut self.slot, || { + *future = None; + }); + } } } -impl<T: 'static, F: Future> Future for TaskLocalFuture<T, F> { - type Output = F::Output; +impl<T: 'static, F> fmt::Debug for TaskLocalFuture<T, F> +where + T: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// Format the Option without Some. + struct TransparentOption<'a, T> { + value: &'a Option<T>, + } + impl<'a, T: fmt::Debug> fmt::Debug for TransparentOption<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.value.as_ref() { + Some(value) => value.fmt(f), + // Hitting the None branch should not be possible. + None => f.pad("<missing>"), + } + } + } - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { - self.with_task(|f| f.poll(cx)) + f.debug_struct("TaskLocalFuture") + .field("value", &TransparentOption { value: &self.slot }) + .finish() } } @@ -300,3 +419,30 @@ impl fmt::Display for AccessError { } impl Error for AccessError {} + +enum ScopeInnerErr { + BorrowError, + AccessError, +} + +impl ScopeInnerErr { + #[track_caller] + fn panic(&self) -> ! { + match self { + Self::BorrowError => panic!("cannot enter a task-local scope while the task-local storage is borrowed"), + Self::AccessError => panic!("cannot enter a task-local scope during or after destruction of the underlying thread-local"), + } + } +} + +impl From<std::cell::BorrowMutError> for ScopeInnerErr { + fn from(_: std::cell::BorrowMutError) -> Self { + Self::BorrowError + } +} + +impl From<std::thread::AccessError> for ScopeInnerErr { + fn from(_: std::thread::AccessError) -> Self { + Self::AccessError + } +} diff --git a/src/task/unconstrained.rs b/src/task/unconstrained.rs index 31c732b..40384c8 100644 --- a/src/task/unconstrained.rs +++ b/src/task/unconstrained.rs @@ -22,7 +22,7 @@ where cfg_coop! { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { let inner = self.project().inner; - crate::coop::with_unconstrained(|| inner.poll(cx)) + crate::runtime::coop::with_unconstrained(|| inner.poll(cx)) } } diff --git a/src/task/yield_now.rs b/src/task/yield_now.rs index 5eeb46a..7b61dd8 100644 --- a/src/task/yield_now.rs +++ b/src/task/yield_now.rs @@ -1,3 +1,5 @@ +use crate::runtime::context; + use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; @@ -33,7 +35,6 @@ use std::task::{Context, Poll}; /// which order the runtime polls your tasks in. /// /// [`tokio::select!`]: macro@crate::select -#[must_use = "yield_now does nothing unless polled/`await`-ed"] #[cfg_attr(docsrs, doc(cfg(feature = "rt")))] pub async fn yield_now() { /// Yield implementation @@ -50,7 +51,17 @@ pub async fn yield_now() { } self.yielded = true; - cx.waker().wake_by_ref(); + + let defer = context::with_defer(|rt| { + rt.defer(cx.waker().clone()); + }); + + if defer.is_none() { + // Not currently in a runtime, just notify ourselves + // immediately. + cx.waker().wake_by_ref(); + } + Poll::Pending } } diff --git a/src/time/clock.rs b/src/time/clock.rs index 41be9ba..cd11a67 100644 --- a/src/time/clock.rs +++ b/src/time/clock.rs @@ -33,7 +33,13 @@ cfg_test_util! { cfg_rt! { fn clock() -> Option<Clock> { - crate::runtime::context::clock() + use crate::runtime::Handle; + + match Handle::try_current() { + Ok(handle) => Some(handle.inner.driver().clock().clone()), + Err(ref e) if e.is_missing_context() => None, + Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + } } } @@ -59,6 +65,9 @@ cfg_test_util! { /// Instant at which the clock was last unfrozen. unfrozen: Option<std::time::Instant>, + + /// Number of `inhibit_auto_advance` calls still in effect. + auto_advance_inhibit_count: usize, } /// Pauses time. @@ -96,6 +105,7 @@ cfg_test_util! { /// /// [`Sleep`]: crate::time::Sleep /// [`advance`]: crate::time::advance + #[track_caller] pub fn pause() { let clock = clock().expect("time cannot be frozen from outside the Tokio runtime"); clock.pause(); @@ -110,6 +120,7 @@ cfg_test_util! { /// /// Panics if time is not frozen or if called from outside of the Tokio /// runtime. + #[track_caller] pub fn resume() { let clock = clock().expect("time cannot be frozen from outside the Tokio runtime"); let mut inner = clock.inner.lock(); @@ -179,6 +190,7 @@ cfg_test_util! { enable_pausing, base: now, unfrozen: Some(now), + auto_advance_inhibit_count: 0, })), }; @@ -189,6 +201,7 @@ cfg_test_util! { clock } + #[track_caller] pub(crate) fn pause(&self) { let mut inner = self.inner.lock(); @@ -203,11 +216,23 @@ cfg_test_util! { inner.unfrozen = None; } - pub(crate) fn is_paused(&self) -> bool { + /// Temporarily stop auto-advancing the clock (see `tokio::time::pause`). + pub(crate) fn inhibit_auto_advance(&self) { + let mut inner = self.inner.lock(); + inner.auto_advance_inhibit_count += 1; + } + + pub(crate) fn allow_auto_advance(&self) { + let mut inner = self.inner.lock(); + inner.auto_advance_inhibit_count -= 1; + } + + pub(crate) fn can_auto_advance(&self) -> bool { let inner = self.inner.lock(); - inner.unfrozen.is_none() + inner.unfrozen.is_none() && inner.auto_advance_inhibit_count == 0 } + #[track_caller] pub(crate) fn advance(&self, duration: Duration) { let mut inner = self.inner.lock(); diff --git a/src/time/driver/handle.rs b/src/time/driver/handle.rs deleted file mode 100644 index 7aaf65a..0000000 --- a/src/time/driver/handle.rs +++ /dev/null @@ -1,88 +0,0 @@ -use crate::loom::sync::Arc; -use crate::time::driver::ClockTime; -use std::fmt; - -/// Handle to time driver instance. -#[derive(Clone)] -pub(crate) struct Handle { - time_source: ClockTime, - inner: Arc<super::Inner>, -} - -impl Handle { - /// Creates a new timer `Handle` from a shared `Inner` timer state. - pub(super) fn new(inner: Arc<super::Inner>) -> Self { - let time_source = inner.state.lock().time_source.clone(); - Handle { time_source, inner } - } - - /// Returns the time source associated with this handle. - pub(super) fn time_source(&self) -> &ClockTime { - &self.time_source - } - - /// Access the driver's inner structure. - pub(super) fn get(&self) -> &super::Inner { - &*self.inner - } - - /// Checks whether the driver has been shutdown. - pub(super) fn is_shutdown(&self) -> bool { - self.inner.is_shutdown() - } -} - -cfg_rt! { - impl Handle { - /// Tries to get a handle to the current timer. - /// - /// # Panics - /// - /// This function panics if there is no current timer set. - /// - /// It can be triggered when `Builder::enable_time()` or - /// `Builder::enable_all()` are not included in the builder. - /// - /// It can also panic whenever a timer is created outside of a - /// Tokio runtime. That is why `rt.block_on(delay_for(...))` will panic, - /// since the function is executed outside of the runtime. - /// Whereas `rt.block_on(async {delay_for(...).await})` doesn't panic. - /// And this is because wrapping the function on an async makes it lazy, - /// and so gets executed inside the runtime successfully without - /// panicking. - pub(crate) fn current() -> Self { - crate::runtime::context::time_handle() - .expect("A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers.") - } - } -} - -cfg_not_rt! { - impl Handle { - /// Tries to get a handle to the current timer. - /// - /// # Panics - /// - /// This function panics if there is no current timer set. - /// - /// It can be triggered when `Builder::enable_time()` or - /// `Builder::enable_all()` are not included in the builder. - /// - /// It can also panic whenever a timer is created outside of a Tokio - /// runtime. That is why `rt.block_on(delay_for(...))` will panic, - /// since the function is executed outside of the runtime. - /// Whereas `rt.block_on(async {delay_for(...).await})` doesn't - /// panic. And this is because wrapping the function on an async makes it - /// lazy, and so outside executed inside the runtime successfully without - /// panicking. - pub(crate) fn current() -> Self { - panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) - } - } -} - -impl fmt::Debug for Handle { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Handle") - } -} diff --git a/src/time/driver/wheel/stack.rs b/src/time/driver/wheel/stack.rs deleted file mode 100644 index 80651c3..0000000 --- a/src/time/driver/wheel/stack.rs +++ /dev/null @@ -1,112 +0,0 @@ -use super::{Item, OwnedItem}; -use crate::time::driver::Entry; - -use std::ptr; - -/// A doubly linked stack. -#[derive(Debug)] -pub(crate) struct Stack { - head: Option<OwnedItem>, -} - -impl Default for Stack { - fn default() -> Stack { - Stack { head: None } - } -} - -impl Stack { - pub(crate) fn is_empty(&self) -> bool { - self.head.is_none() - } - - pub(crate) fn push(&mut self, entry: OwnedItem) { - // Get a pointer to the entry to for the prev link - let ptr: *const Entry = &*entry as *const _; - - // Remove the old head entry - let old = self.head.take(); - - unsafe { - // Ensure the entry is not already in a stack. - debug_assert!((*entry.next_stack.get()).is_none()); - debug_assert!((*entry.prev_stack.get()).is_null()); - - if let Some(ref entry) = old.as_ref() { - debug_assert!({ - // The head is not already set to the entry - ptr != &***entry as *const _ - }); - - // Set the previous link on the old head - *entry.prev_stack.get() = ptr; - } - - // Set this entry's next pointer - *entry.next_stack.get() = old; - } - - // Update the head pointer - self.head = Some(entry); - } - - /// Pops an item from the stack. - pub(crate) fn pop(&mut self) -> Option<OwnedItem> { - let entry = self.head.take(); - - unsafe { - if let Some(entry) = entry.as_ref() { - self.head = (*entry.next_stack.get()).take(); - - if let Some(entry) = self.head.as_ref() { - *entry.prev_stack.get() = ptr::null(); - } - - *entry.prev_stack.get() = ptr::null(); - } - } - - entry - } - - pub(crate) fn remove(&mut self, entry: &Item) { - unsafe { - // Ensure that the entry is in fact contained by the stack - debug_assert!({ - // This walks the full linked list even if an entry is found. - let mut next = self.head.as_ref(); - let mut contains = false; - - while let Some(n) = next { - if entry as *const _ == &**n as *const _ { - debug_assert!(!contains); - contains = true; - } - - next = (*n.next_stack.get()).as_ref(); - } - - contains - }); - - // Unlink `entry` from the next node - let next = (*entry.next_stack.get()).take(); - - if let Some(next) = next.as_ref() { - (*next.prev_stack.get()) = *entry.prev_stack.get(); - } - - // Unlink `entry` from the prev node - - if let Some(prev) = (*entry.prev_stack.get()).as_ref() { - *prev.next_stack.get() = next; - } else { - // It is the head - self.head = next; - } - - // Unset the prev pointer - *entry.prev_stack.get() = ptr::null(); - } - } -} diff --git a/src/time/error.rs b/src/time/error.rs index 63f0a3b..71344d4 100644 --- a/src/time/error.rs +++ b/src/time/error.rs @@ -41,7 +41,10 @@ impl From<Kind> for Error { } /// Errors returned by `Timeout`. -#[derive(Debug, PartialEq)] +/// +/// This error is returned when a timeout expires before the function was able +/// to finish. +#[derive(Debug, PartialEq, Eq)] pub struct Elapsed(()); #[derive(Debug)] diff --git a/src/time/instant.rs b/src/time/instant.rs index f7cf12d..f184929 100644 --- a/src/time/instant.rs +++ b/src/time/instant.rs @@ -67,13 +67,10 @@ impl Instant { self.std } - /// Returns the amount of time elapsed from another instant to this one. - /// - /// # Panics - /// - /// This function will panic if `earlier` is later than `self`. + /// Returns the amount of time elapsed from another instant to this one, or + /// zero duration if that instant is later than this one. pub fn duration_since(&self, earlier: Instant) -> Duration { - self.std.duration_since(earlier.std) + self.std.saturating_duration_since(earlier.std) } /// Returns the amount of time elapsed from another instant to this one, or @@ -118,13 +115,8 @@ impl Instant { self.std.saturating_duration_since(earlier.std) } - /// Returns the amount of time elapsed since this instant was created. - /// - /// # Panics - /// - /// This function may panic if the current time is earlier than this - /// instant, which is something that can happen if an `Instant` is - /// produced synthetically. + /// Returns the amount of time elapsed since this instant was created, + /// or zero duration if that this instant is in the future. /// /// # Examples /// @@ -140,7 +132,7 @@ impl Instant { /// } /// ``` pub fn elapsed(&self) -> Duration { - Instant::now() - *self + Instant::now().saturating_duration_since(*self) } /// Returns `Some(t)` where `t` is the time `self + duration` if `t` can be @@ -188,7 +180,7 @@ impl ops::Sub for Instant { type Output = Duration; fn sub(self, rhs: Instant) -> Duration { - self.std - rhs.std + self.std.saturating_duration_since(rhs.std) } } diff --git a/src/time/interval.rs b/src/time/interval.rs index 7e07e51..ea8b393 100644 --- a/src/time/interval.rs +++ b/src/time/interval.rs @@ -1,6 +1,8 @@ use crate::future::poll_fn; use crate::time::{sleep_until, Duration, Instant, Sleep}; +use crate::util::trace; +use std::panic::Location; use std::pin::Pin; use std::task::{Context, Poll}; use std::{convert::TryInto, future::Future}; @@ -68,10 +70,10 @@ use std::{convert::TryInto, future::Future}; /// /// [`sleep`]: crate::time::sleep() /// [`.tick().await`]: Interval::tick +#[track_caller] pub fn interval(period: Duration) -> Interval { assert!(period > Duration::new(0, 0), "`period` must be non-zero."); - - interval_at(Instant::now(), period) + internal_interval_at(Instant::now(), period, trace::caller_location()) } /// Creates new [`Interval`] that yields with interval of `period` with the @@ -103,13 +105,44 @@ pub fn interval(period: Duration) -> Interval { /// // approximately 70ms have elapsed. /// } /// ``` +#[track_caller] pub fn interval_at(start: Instant, period: Duration) -> Interval { assert!(period > Duration::new(0, 0), "`period` must be non-zero."); + internal_interval_at(start, period, trace::caller_location()) +} + +#[cfg_attr(not(all(tokio_unstable, feature = "tracing")), allow(unused_variables))] +fn internal_interval_at( + start: Instant, + period: Duration, + location: Option<&'static Location<'static>>, +) -> Interval { + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = { + let location = location.expect("should have location if tracing"); + + tracing::trace_span!( + "runtime.resource", + concrete_type = "Interval", + kind = "timer", + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + ) + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let delay = resource_span.in_scope(|| Box::pin(sleep_until(start))); + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let delay = Box::pin(sleep_until(start)); Interval { - delay: Box::pin(sleep_until(start)), + delay, period, missed_tick_behavior: Default::default(), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span, } } @@ -124,7 +157,7 @@ pub fn interval_at(start: Instant, period: Duration) -> Interval { /// /// #[tokio::main] /// async fn main() { -/// // ticks every 2 seconds +/// // ticks every 2 milliseconds /// let mut interval = time::interval(Duration::from_millis(2)); /// for _ in 0..5 { /// interval.tick().await; @@ -174,6 +207,9 @@ pub enum MissedTickBehavior { /// # async fn main() { /// let mut interval = interval(Duration::from_millis(50)); /// + /// // First tick resolves immediately after creation + /// interval.tick().await; + /// /// task_that_takes_200_millis().await; /// // The `Interval` has missed a tick /// @@ -242,7 +278,7 @@ pub enum MissedTickBehavior { /// // 50ms after the call to `tick` up above. That is, in `tick`, when we /// // recognize that we missed a tick, we schedule the next tick to happen /// // 50ms (or whatever the `period` is) from right then, not from when - /// // were were *supposed* to tick + /// // were *supposed* to tick /// interval.tick().await; /// # } /// ``` @@ -362,6 +398,9 @@ pub struct Interval { /// The strategy `Interval` should use when a tick is missed. missed_tick_behavior: MissedTickBehavior, + + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, } impl Interval { @@ -384,6 +423,7 @@ impl Interval { /// let mut interval = time::interval(Duration::from_millis(10)); /// /// interval.tick().await; + /// // approximately 0ms have elapsed. The first tick completes immediately. /// interval.tick().await; /// interval.tick().await; /// @@ -391,7 +431,20 @@ impl Interval { /// } /// ``` pub async fn tick(&mut self) -> Instant { - poll_fn(|cx| self.poll_tick(cx)).await + #[cfg(all(tokio_unstable, feature = "tracing"))] + let resource_span = self.resource_span.clone(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let instant = trace::async_op( + || poll_fn(|cx| self.poll_tick(cx)), + resource_span, + "Interval::tick", + "poll_tick", + false, + ); + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let instant = poll_fn(|cx| self.poll_tick(cx)); + + instant.await } /// Polls for the next instant in the interval to be reached. @@ -435,6 +488,36 @@ impl Interval { Poll::Ready(timeout) } + /// Resets the interval to complete one period after the current time. + /// + /// This method ignores [`MissedTickBehavior`] strategy. + /// + /// # Examples + /// + /// ``` + /// use tokio::time; + /// + /// use std::time::Duration; + /// + /// #[tokio::main] + /// async fn main() { + /// let mut interval = time::interval(Duration::from_millis(100)); + /// + /// interval.tick().await; + /// + /// time::sleep(Duration::from_millis(50)).await; + /// interval.reset(); + /// + /// interval.tick().await; + /// interval.tick().await; + /// + /// // approximately 250ms have elapsed. + /// } + /// ``` + pub fn reset(&mut self) { + self.delay.as_mut().reset(Instant::now() + self.period); + } + /// Returns the [`MissedTickBehavior`] strategy currently being used. pub fn missed_tick_behavior(&self) -> MissedTickBehavior { self.missed_tick_behavior diff --git a/src/time/mod.rs b/src/time/mod.rs index 281990e..a1f27b8 100644 --- a/src/time/mod.rs +++ b/src/time/mod.rs @@ -82,17 +82,13 @@ //! ``` //! //! [`interval`]: crate::time::interval() +//! [`sleep`]: sleep() mod clock; pub(crate) use self::clock::Clock; #[cfg(feature = "test-util")] pub use clock::{advance, pause, resume}; -pub(crate) mod driver; - -#[doc(inline)] -pub use driver::sleep::{sleep, sleep_until, Sleep}; - pub mod error; mod instant; @@ -101,14 +97,13 @@ pub use self::instant::Instant; mod interval; pub use interval::{interval, interval_at, Interval, MissedTickBehavior}; +mod sleep; +pub use sleep::{sleep, sleep_until, Sleep}; + mod timeout; #[doc(inline)] pub use timeout::{timeout, timeout_at, Timeout}; -#[cfg(test)] -#[cfg(not(loom))] -mod tests; - // Re-export for convenience #[doc(no_inline)] pub use std::time::Duration; diff --git a/src/time/driver/sleep.rs b/src/time/sleep.rs index 43ff694..0a012e2 100644 --- a/src/time/driver/sleep.rs +++ b/src/time/sleep.rs @@ -1,4 +1,4 @@ -use crate::time::driver::{Handle, TimerEntry}; +use crate::runtime::time::TimerEntry; use crate::time::{error::Error, Duration, Instant}; use crate::util::trace; @@ -8,10 +8,6 @@ use std::panic::Location; use std::pin::Pin; use std::task::{self, Poll}; -cfg_trace! { - use crate::time::driver::ClockTime; -} - /// Waits until `deadline` is reached. /// /// No work is performed while awaiting on the sleep future to complete. `Sleep` @@ -41,11 +37,28 @@ cfg_trace! { /// /// See the documentation for the [`Sleep`] type for more examples. /// +/// # Panics +/// +/// This function panics if there is no current timer set. +/// +/// It can be triggered when [`Builder::enable_time`] or +/// [`Builder::enable_all`] are not included in the builder. +/// +/// It can also panic whenever a timer is created outside of a +/// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic, +/// since the function is executed outside of the runtime. +/// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic. +/// And this is because wrapping the function on an async makes it lazy, +/// and so gets executed inside the runtime successfully without +/// panicking. +/// /// [`Sleep`]: struct@crate::time::Sleep /// [`interval`]: crate::time::interval() +/// [`Builder::enable_time`]: crate::runtime::Builder::enable_time +/// [`Builder::enable_all`]: crate::runtime::Builder::enable_all // Alias for old name in 0.x #[cfg_attr(docsrs, doc(alias = "delay_until"))] -#[cfg_attr(tokio_track_caller, track_caller)] +#[track_caller] pub fn sleep_until(deadline: Instant) -> Sleep { return Sleep::new_timeout(deadline, trace::caller_location()); } @@ -57,7 +70,9 @@ pub fn sleep_until(deadline: Instant) -> Sleep { /// /// No work is performed while awaiting on the sleep future to complete. `Sleep` /// operates at millisecond granularity and should not be used for tasks that -/// require high-resolution timers. +/// require high-resolution timers. The implementation is platform specific, +/// and some platforms (specifically Windows) will provide timers with a +/// larger resolution than 1 ms. /// /// To run something regularly on a schedule, see [`interval`]. /// @@ -84,12 +99,29 @@ pub fn sleep_until(deadline: Instant) -> Sleep { /// /// See the documentation for the [`Sleep`] type for more examples. /// +/// # Panics +/// +/// This function panics if there is no current timer set. +/// +/// It can be triggered when [`Builder::enable_time`] or +/// [`Builder::enable_all`] are not included in the builder. +/// +/// It can also panic whenever a timer is created outside of a +/// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic, +/// since the function is executed outside of the runtime. +/// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic. +/// And this is because wrapping the function on an async makes it lazy, +/// and so gets executed inside the runtime successfully without +/// panicking. +/// /// [`Sleep`]: struct@crate::time::Sleep /// [`interval`]: crate::time::interval() +/// [`Builder::enable_time`]: crate::runtime::Builder::enable_time +/// [`Builder::enable_all`]: crate::runtime::Builder::enable_all // Alias for old name in 0.x #[cfg_attr(docsrs, doc(alias = "delay_for"))] #[cfg_attr(docsrs, doc(alias = "wait"))] -#[cfg_attr(tokio_track_caller, track_caller)] +#[track_caller] pub fn sleep(duration: Duration) -> Sleep { let location = trace::caller_location(); @@ -204,9 +236,7 @@ cfg_trace! { #[derive(Debug)] struct Inner { deadline: Instant, - resource_span: tracing::Span, - async_op_span: tracing::Span, - time_source: ClockTime, + ctx: trace::AsyncOpTracingCtx, } } @@ -219,23 +249,24 @@ cfg_not_trace! { impl Sleep { #[cfg_attr(not(all(tokio_unstable, feature = "tracing")), allow(unused_variables))] + #[track_caller] pub(crate) fn new_timeout( deadline: Instant, location: Option<&'static Location<'static>>, ) -> Sleep { - let handle = Handle::current(); + use crate::runtime::scheduler; + + let handle = scheduler::Handle::current(); let entry = TimerEntry::new(&handle, deadline); #[cfg(all(tokio_unstable, feature = "tracing"))] let inner = { - let time_source = handle.time_source().clone(); + let handle = &handle.driver().time(); + let time_source = handle.time_source(); let deadline_tick = time_source.deadline_to_tick(deadline); - let duration = deadline_tick.checked_sub(time_source.now()).unwrap_or(0); + let duration = deadline_tick.saturating_sub(time_source.now()); - #[cfg(tokio_track_caller)] - let location = location.expect("should have location if tracking caller"); - - #[cfg(tokio_track_caller)] + let location = location.expect("should have location if tracing"); let resource_span = tracing::trace_span!( "runtime.resource", concrete_type = "Sleep", @@ -245,27 +276,27 @@ impl Sleep { loc.col = location.column(), ); - #[cfg(not(tokio_track_caller))] - let resource_span = - tracing::trace_span!("runtime.resource", concrete_type = "Sleep", kind = "timer"); + let async_op_span = resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + duration = duration, + duration.unit = "ms", + duration.op = "override", + ); - let async_op_span = - tracing::trace_span!("runtime.resource.async_op", source = "Sleep::new_timeout"); + tracing::trace_span!("runtime.resource.async_op", source = "Sleep::new_timeout") + }); - tracing::trace!( - target: "runtime::resource::state_update", - parent: resource_span.id(), - duration = duration, - duration.unit = "ms", - duration.op = "override", - ); + let async_op_poll_span = + async_op_span.in_scope(|| tracing::trace_span!("runtime.resource.async_op.poll")); - Inner { - deadline, - resource_span, + let ctx = trace::AsyncOpTracingCtx { async_op_span, - time_source, - } + async_op_poll_span, + resource_span, + }; + + Inner { deadline, ctx } }; #[cfg(not(all(tokio_unstable, feature = "tracing")))] @@ -324,60 +355,59 @@ impl Sleep { } fn reset_inner(self: Pin<&mut Self>, deadline: Instant) { - let me = self.project(); - me.entry.reset(deadline); - (*me.inner).deadline = deadline; + let mut me = self.project(); + me.entry.as_mut().reset(deadline); + (me.inner).deadline = deadline; #[cfg(all(tokio_unstable, feature = "tracing"))] { - me.inner.async_op_span = + let _resource_enter = me.inner.ctx.resource_span.enter(); + me.inner.ctx.async_op_span = tracing::trace_span!("runtime.resource.async_op", source = "Sleep::reset"); + let _async_op_enter = me.inner.ctx.async_op_span.enter(); + + me.inner.ctx.async_op_poll_span = + tracing::trace_span!("runtime.resource.async_op.poll"); + + let duration = { + let time_source = me.entry.driver().time_source(); + let now = time_source.now(); + let deadline_tick = time_source.deadline_to_tick(deadline); + deadline_tick.saturating_sub(now) + }; tracing::trace!( target: "runtime::resource::state_update", - parent: me.inner.resource_span.id(), - duration = { - let now = me.inner.time_source.now(); - let deadline_tick = me.inner.time_source.deadline_to_tick(deadline); - deadline_tick.checked_sub(now).unwrap_or(0) - }, + duration = duration, duration.unit = "ms", duration.op = "override", ); } } - cfg_not_trace! { - fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Result<(), Error>> { - let me = self.project(); + fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Result<(), Error>> { + let me = self.project(); - // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + // Keep track of task budget + #[cfg(all(tokio_unstable, feature = "tracing"))] + let coop = ready!(trace_poll_op!( + "poll_elapsed", + crate::runtime::coop::poll_proceed(cx), + )); - me.entry.poll_elapsed(cx).map(move |r| { - coop.made_progress(); - r - }) - } - } + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); - cfg_trace! { - fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Result<(), Error>> { - let me = self.project(); - // Keep track of task budget - let coop = ready!(trace_poll_op!( - "poll_elapsed", - crate::coop::poll_proceed(cx), - me.inner.resource_span.id(), - )); - - let result = me.entry.poll_elapsed(cx).map(move |r| { - coop.made_progress(); - r - }); + let result = me.entry.poll_elapsed(cx).map(move |r| { + coop.made_progress(); + r + }); - trace_poll_op!("poll_elapsed", result, me.inner.resource_span.id()) - } + #[cfg(all(tokio_unstable, feature = "tracing"))] + return trace_poll_op!("poll_elapsed", result); + + #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] + return result; } } @@ -395,8 +425,11 @@ impl Future for Sleep { // really do much better if we passed the error onwards. fn poll(mut self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> { #[cfg(all(tokio_unstable, feature = "tracing"))] - let _span = self.inner.async_op_span.clone().entered(); - + let _res_span = self.inner.ctx.resource_span.clone().entered(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _ao_span = self.inner.ctx.async_op_span.clone().entered(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let _ao_poll_span = self.inner.ctx.async_op_poll_span.clone().entered(); match ready!(self.as_mut().poll_elapsed(cx)) { Ok(()) => Poll::Ready(()), Err(e) => panic!("timer error: {}", e), diff --git a/src/time/tests/mod.rs b/src/time/tests/mod.rs deleted file mode 100644 index 35e1060..0000000 --- a/src/time/tests/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -mod test_sleep; - -use crate::time::{self, Instant}; -use std::time::Duration; - -fn assert_send<T: Send>() {} -fn assert_sync<T: Sync>() {} - -#[test] -fn registration_is_send_and_sync() { - use crate::time::Sleep; - - assert_send::<Sleep>(); - assert_sync::<Sleep>(); -} - -#[test] -#[should_panic] -fn sleep_is_eager() { - let when = Instant::now() + Duration::from_millis(100); - let _ = time::sleep_until(when); -} diff --git a/src/time/tests/test_sleep.rs b/src/time/tests/test_sleep.rs deleted file mode 100644 index 77ca07e..0000000 --- a/src/time/tests/test_sleep.rs +++ /dev/null @@ -1,443 +0,0 @@ -//use crate::time::driver::{Driver, Entry, Handle}; - -/* -macro_rules! poll { - ($e:expr) => { - $e.enter(|cx, e| e.poll_elapsed(cx)) - }; -} - -#[test] -fn frozen_utility_returns_correct_advanced_duration() { - let clock = Clock::new(); - clock.pause(); - let start = clock.now(); - - clock.advance(ms(10)); - assert_eq!(clock.now() - start, ms(10)); -} - -#[test] -fn immediate_sleep() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - let when = clock.now(); - let mut e = task::spawn(sleep_until(&handle, when)); - - assert_ready_ok!(poll!(e)); - - assert_ok!(driver.park_timeout(Duration::from_millis(1000))); - - // The time has not advanced. The `turn` completed immediately. - assert_eq!(clock.now() - start, ms(1000)); -} - -#[test] -fn delayed_sleep_level_0() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - for &i in &[1, 10, 60] { - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, start + ms(i))); - - // The sleep instance has not elapsed. - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(i)); - - assert_ready_ok!(poll!(e)); - } -} - -#[test] -fn sub_ms_delayed_sleep() { - let (mut driver, clock, handle) = setup(); - - for _ in 0..5 { - let deadline = clock.now() + ms(1) + Duration::new(0, 1); - - let mut e = task::spawn(sleep_until(&handle, deadline)); - - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_ready_ok!(poll!(e)); - - assert!(clock.now() >= deadline); - - clock.advance(Duration::new(0, 1)); - } -} - -#[test] -fn delayed_sleep_wrapping_level_0() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - assert_ok!(driver.park_timeout(ms(5))); - assert_eq!(clock.now() - start, ms(5)); - - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(60))); - - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(64)); - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(65)); - - assert_ready_ok!(poll!(e)); -} - -#[test] -fn timer_wrapping_with_higher_levels() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Set sleep to hit level 1 - let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(64))); - assert_pending!(poll!(e1)); - - // Turn a bit - assert_ok!(driver.park_timeout(ms(5))); - - // Set timeout such that it will hit level 0, but wrap - let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(60))); - assert_pending!(poll!(e2)); - - // This should result in s1 firing - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(64)); - - assert_ready_ok!(poll!(e1)); - assert_pending!(poll!(e2)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(65)); - - assert_ready_ok!(poll!(e1)); -} - -#[test] -fn sleep_with_deadline_in_past() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create `Sleep` that elapsed immediately. - let mut e = task::spawn(sleep_until(&handle, clock.now() - ms(100))); - - // Even though the `Sleep` expires in the past, it is not ready yet - // because the timer must observe it. - assert_ready_ok!(poll!(e)); - - // Turn the timer, it runs for the elapsed time - assert_ok!(driver.park_timeout(ms(1000))); - - // The time has not advanced. The `turn` completed immediately. - assert_eq!(clock.now() - start, ms(1000)); -} - -#[test] -fn delayed_sleep_level_1() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(234))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer, this will wake up to cascade the timer down. - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(192)); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer again - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(234)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(234))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer with a smaller timeout than the cascade. - assert_ok!(driver.park_timeout(ms(100))); - assert_eq!(clock.now() - start, ms(100)); - - assert_pending!(poll!(e)); - - // Turn the timer, this will wake up to cascade the timer down. - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(192)); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer again - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(234)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); -} - -#[test] -fn concurrently_set_two_timers_second_one_shorter() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(500))); - let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(200))); - - // The sleep has not elapsed - assert_pending!(poll!(e1)); - assert_pending!(poll!(e2)); - - // Sleep until a cascade - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(192)); - - // Sleep until the second timer. - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(200)); - - // The shorter sleep fires - assert_ready_ok!(poll!(e2)); - assert_pending!(poll!(e1)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(448)); - - assert_pending!(poll!(e1)); - - // Turn again, this time the time will advance to the second sleep - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(500)); - - assert_ready_ok!(poll!(e1)); -} - -#[test] -fn short_sleep() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(1))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer, but not enough time will go by. - assert_ok!(driver.park()); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); - - // The time has advanced to the point of the sleep elapsing. - assert_eq!(clock.now() - start, ms(1)); -} - -#[test] -fn sorta_long_sleep_until() { - const MIN_5: u64 = 5 * 60 * 1000; - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(MIN_5))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - let cascades = &[262_144, 262_144 + 9 * 4096, 262_144 + 9 * 4096 + 15 * 64]; - - for &elapsed in cascades { - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(elapsed)); - - assert_pending!(poll!(e)); - } - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(MIN_5)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); -} - -#[test] -fn very_long_sleep() { - const MO_5: u64 = 5 * 30 * 24 * 60 * 60 * 1000; - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(MO_5))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - let cascades = &[ - 12_884_901_888, - 12_952_010_752, - 12_959_875_072, - 12_959_997_952, - ]; - - for &elapsed in cascades { - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(elapsed)); - - assert_pending!(poll!(e)); - } - - // Turn the timer, but not enough time will go by. - assert_ok!(driver.park()); - - // The time has advanced to the point of the sleep elapsing. - assert_eq!(clock.now() - start, ms(MO_5)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); -} - -#[test] -fn unpark_is_delayed() { - // A special park that will take much longer than the requested duration - struct MockPark(Clock); - - struct MockUnpark; - - impl Park for MockPark { - type Unpark = MockUnpark; - type Error = (); - - fn unpark(&self) -> Self::Unpark { - MockUnpark - } - - fn park(&mut self) -> Result<(), Self::Error> { - panic!("parking forever"); - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - assert_eq!(duration, ms(0)); - self.0.advance(ms(436)); - Ok(()) - } - - fn shutdown(&mut self) {} - } - - impl Unpark for MockUnpark { - fn unpark(&self) {} - } - - let clock = Clock::new(); - clock.pause(); - let start = clock.now(); - let mut driver = Driver::new(MockPark(clock.clone()), clock.clone()); - let handle = driver.handle(); - - let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(100))); - let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(101))); - let mut e3 = task::spawn(sleep_until(&handle, clock.now() + ms(200))); - - assert_pending!(poll!(e1)); - assert_pending!(poll!(e2)); - assert_pending!(poll!(e3)); - - assert_ok!(driver.park()); - - assert_eq!(clock.now() - start, ms(500)); - - assert_ready_ok!(poll!(e1)); - assert_ready_ok!(poll!(e2)); - assert_ready_ok!(poll!(e3)); -} - -#[test] -fn set_timeout_at_deadline_greater_than_max_timer() { - const YR_1: u64 = 365 * 24 * 60 * 60 * 1000; - const YR_5: u64 = 5 * YR_1; - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - for _ in 0..5 { - assert_ok!(driver.park_timeout(ms(YR_1))); - } - - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(1))); - assert_pending!(poll!(e)); - - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(YR_5) + ms(1)); - - assert_ready_ok!(poll!(e)); -} - -fn setup() -> (Driver<MockPark>, Clock, Handle) { - let clock = Clock::new(); - clock.pause(); - let driver = Driver::new(MockPark(clock.clone()), clock.clone()); - let handle = driver.handle(); - - (driver, clock, handle) -} - -fn sleep_until(handle: &Handle, when: Instant) -> Arc<Entry> { - Entry::new(&handle, when, ms(0)) -} - -struct MockPark(Clock); - -struct MockUnpark; - -impl Park for MockPark { - type Unpark = MockUnpark; - type Error = (); - - fn unpark(&self) -> Self::Unpark { - MockUnpark - } - - fn park(&mut self) -> Result<(), Self::Error> { - panic!("parking forever"); - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.0.advance(duration); - Ok(()) - } - - fn shutdown(&mut self) {} -} - -impl Unpark for MockUnpark { - fn unpark(&self) {} -} - -fn ms(n: u64) -> Duration { - Duration::from_millis(n) -} -*/ diff --git a/src/time/timeout.rs b/src/time/timeout.rs index 6725caa..3bb98ea 100644 --- a/src/time/timeout.rs +++ b/src/time/timeout.rs @@ -5,6 +5,7 @@ //! [`Timeout`]: struct@Timeout use crate::{ + runtime::coop, time::{error::Elapsed, sleep_until, Duration, Instant, Sleep}, util::trace, }; @@ -20,7 +21,17 @@ use std::task::{self, Poll}; /// value is returned. Otherwise, an error is returned and the future is /// canceled. /// -/// # Cancelation +/// Note that the timeout is checked before polling the future, so if the future +/// does not yield during execution then it is possible for the future to complete +/// and exceed the timeout _without_ returning an error. +/// +/// This function returns a future whose return type is [`Result`]`<T,`[`Elapsed`]`>`, where `T` is the +/// return type of the provided future. +/// +/// [`Result`]: std::result::Result +/// [`Elapsed`]: crate::time::error::Elapsed +/// +/// # Cancellation /// /// Cancelling a timeout is done by dropping the future. No additional cleanup /// or other work is required. @@ -48,10 +59,28 @@ use std::task::{self, Poll}; /// } /// # } /// ``` -#[cfg_attr(tokio_track_caller, track_caller)] -pub fn timeout<T>(duration: Duration, future: T) -> Timeout<T> +/// +/// # Panics +/// +/// This function panics if there is no current timer set. +/// +/// It can be triggered when [`Builder::enable_time`] or +/// [`Builder::enable_all`] are not included in the builder. +/// +/// It can also panic whenever a timer is created outside of a +/// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic, +/// since the function is executed outside of the runtime. +/// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic. +/// And this is because wrapping the function on an async makes it lazy, +/// and so gets executed inside the runtime successfully without +/// panicking. +/// +/// [`Builder::enable_time`]: crate::runtime::Builder::enable_time +/// [`Builder::enable_all`]: crate::runtime::Builder::enable_all +#[track_caller] +pub fn timeout<F>(duration: Duration, future: F) -> Timeout<F> where - T: Future, + F: Future, { let location = trace::caller_location(); @@ -68,7 +97,13 @@ where /// If the future completes before the instant is reached, then the completed /// value is returned. Otherwise, an error is returned. /// -/// # Cancelation +/// This function returns a future whose return type is [`Result`]`<T,`[`Elapsed`]`>`, where `T` is the +/// return type of the provided future. +/// +/// [`Result`]: std::result::Result +/// [`Elapsed`]: crate::time::error::Elapsed +/// +/// # Cancellation /// /// Cancelling a timeout is done by dropping the future. No additional cleanup /// or other work is required. @@ -97,9 +132,9 @@ where /// } /// # } /// ``` -pub fn timeout_at<T>(deadline: Instant, future: T) -> Timeout<T> +pub fn timeout_at<F>(deadline: Instant, future: F) -> Timeout<F> where - T: Future, + F: Future, { let delay = sleep_until(deadline); @@ -151,15 +186,33 @@ where fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> { let me = self.project(); + let had_budget_before = coop::has_budget_remaining(); + // First, try polling the future if let Poll::Ready(v) = me.value.poll(cx) { return Poll::Ready(Ok(v)); } - // Now check the timer - match me.delay.poll(cx) { - Poll::Ready(()) => Poll::Ready(Err(Elapsed::new())), - Poll::Pending => Poll::Pending, + let has_budget_now = coop::has_budget_remaining(); + + let delay = me.delay; + + let poll_delay = || -> Poll<Self::Output> { + match delay.poll(cx) { + Poll::Ready(()) => Poll::Ready(Err(Elapsed::new())), + Poll::Pending => Poll::Pending, + } + }; + + if let (true, false) = (had_budget_before, has_budget_now) { + // if it is the underlying future that exhausted the budget, we poll + // the `delay` with an unconstrained one. This prevents pathological + // cases where the underlying future always exhausts the budget and + // we never get a chance to evaluate whether the timeout was hit or + // not. + coop::with_unconstrained(poll_delay) + } else { + poll_delay() } } } diff --git a/src/runtime/thread_pool/atomic_cell.rs b/src/util/atomic_cell.rs index 98847e6..07e3730 100644 --- a/src/runtime/thread_pool/atomic_cell.rs +++ b/src/util/atomic_cell.rs @@ -3,7 +3,7 @@ use crate::loom::sync::atomic::AtomicPtr; use std::ptr; use std::sync::atomic::Ordering::AcqRel; -pub(super) struct AtomicCell<T> { +pub(crate) struct AtomicCell<T> { data: AtomicPtr<T>, } @@ -11,22 +11,22 @@ unsafe impl<T: Send> Send for AtomicCell<T> {} unsafe impl<T: Send> Sync for AtomicCell<T> {} impl<T> AtomicCell<T> { - pub(super) fn new(data: Option<Box<T>>) -> AtomicCell<T> { + pub(crate) fn new(data: Option<Box<T>>) -> AtomicCell<T> { AtomicCell { data: AtomicPtr::new(to_raw(data)), } } - pub(super) fn swap(&self, val: Option<Box<T>>) -> Option<Box<T>> { + pub(crate) fn swap(&self, val: Option<Box<T>>) -> Option<Box<T>> { let old = self.data.swap(to_raw(val), AcqRel); from_raw(old) } - pub(super) fn set(&self, val: Box<T>) { + pub(crate) fn set(&self, val: Box<T>) { let _ = self.swap(Some(val)); } - pub(super) fn take(&self) -> Option<Box<T>> { + pub(crate) fn take(&self) -> Option<Box<T>> { self.swap(None) } } diff --git a/src/util/error.rs b/src/util/error.rs index 8f252c0..ebb27f6 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -1,15 +1,14 @@ +// Some combinations of features may not use these constants. +#![cfg_attr(not(feature = "full"), allow(dead_code))] + /// Error string explaining that the Tokio context hasn't been instantiated. pub(crate) const CONTEXT_MISSING_ERROR: &str = "there is no reactor running, must be called from the context of a Tokio 1.x runtime"; -// some combinations of features might not use this -#[allow(dead_code)] /// Error string explaining that the Tokio context is shutting down and cannot drive timers. pub(crate) const RUNTIME_SHUTTING_DOWN_ERROR: &str = "A Tokio 1.x context was found, but it is being shutdown."; -// some combinations of features might not use this -#[allow(dead_code)] /// Error string explaining that the Tokio context is not available because the /// thread-local storing it has been destroyed. This usually only happens during /// destructors of other thread-locals. diff --git a/src/util/idle_notified_set.rs b/src/util/idle_notified_set.rs new file mode 100644 index 0000000..ce8ff9e --- /dev/null +++ b/src/util/idle_notified_set.rs @@ -0,0 +1,462 @@ +//! This module defines an `IdleNotifiedSet`, which is a collection of elements. +//! Each element is intended to correspond to a task, and the collection will +//! keep track of which tasks have had their waker notified, and which have not. +//! +//! Each entry in the set holds some user-specified value. The value's type is +//! specified using the `T` parameter. It will usually be a `JoinHandle` or +//! similar. + +use std::marker::PhantomPinned; +use std::mem::ManuallyDrop; +use std::ptr::NonNull; +use std::task::{Context, Waker}; + +use crate::loom::cell::UnsafeCell; +use crate::loom::sync::{Arc, Mutex}; +use crate::util::linked_list::{self, Link}; +use crate::util::{waker_ref, Wake}; + +type LinkedList<T> = + linked_list::LinkedList<ListEntry<T>, <ListEntry<T> as linked_list::Link>::Target>; + +/// This is the main handle to the collection. +pub(crate) struct IdleNotifiedSet<T> { + lists: Arc<Lists<T>>, + length: usize, +} + +/// A handle to an entry that is guaranteed to be stored in the idle or notified +/// list of its `IdleNotifiedSet`. This value borrows the `IdleNotifiedSet` +/// mutably to prevent the entry from being moved to the `Neither` list, which +/// only the `IdleNotifiedSet` may do. +/// +/// The main consequence of being stored in one of the lists is that the `value` +/// field has not yet been consumed. +/// +/// Note: This entry can be moved from the idle to the notified list while this +/// object exists by waking its waker. +pub(crate) struct EntryInOneOfTheLists<'a, T> { + entry: Arc<ListEntry<T>>, + set: &'a mut IdleNotifiedSet<T>, +} + +type Lists<T> = Mutex<ListsInner<T>>; + +/// The linked lists hold strong references to the ListEntry items, and the +/// ListEntry items also hold a strong reference back to the Lists object, but +/// the destructor of the `IdleNotifiedSet` will clear the two lists, so once +/// that object is destroyed, no ref-cycles will remain. +struct ListsInner<T> { + notified: LinkedList<T>, + idle: LinkedList<T>, + /// Whenever an element in the `notified` list is woken, this waker will be + /// notified and consumed, if it exists. + waker: Option<Waker>, +} + +/// Which of the two lists in the shared Lists object is this entry stored in? +/// +/// If the value is `Idle`, then an entry's waker may move it to the notified +/// list. Otherwise, only the `IdleNotifiedSet` may move it. +/// +/// If the value is `Neither`, then it is still possible that the entry is in +/// some third external list (this happens in `drain`). +#[derive(Copy, Clone, Eq, PartialEq)] +enum List { + Notified, + Idle, + Neither, +} + +/// An entry in the list. +/// +/// # Safety +/// +/// The `my_list` field must only be accessed while holding the mutex in +/// `parent`. It is an invariant that the value of `my_list` corresponds to +/// which linked list in the `parent` holds this entry. Once this field takes +/// the value `Neither`, then it may never be modified again. +/// +/// If the value of `my_list` is `Notified` or `Idle`, then the `pointers` field +/// must only be accessed while holding the mutex. If the value of `my_list` is +/// `Neither`, then the `pointers` field may be accessed by the +/// `IdleNotifiedSet` (this happens inside `drain`). +/// +/// The `value` field is owned by the `IdleNotifiedSet` and may only be accessed +/// by the `IdleNotifiedSet`. The operation that sets the value of `my_list` to +/// `Neither` assumes ownership of the `value`, and it must either drop it or +/// move it out from this entry to prevent it from getting leaked. (Since the +/// two linked lists are emptied in the destructor of `IdleNotifiedSet`, the +/// value should not be leaked.) +struct ListEntry<T> { + /// The linked list pointers of the list this entry is in. + pointers: linked_list::Pointers<ListEntry<T>>, + /// Pointer to the shared `Lists` struct. + parent: Arc<Lists<T>>, + /// The value stored in this entry. + value: UnsafeCell<ManuallyDrop<T>>, + /// Used to remember which list this entry is in. + my_list: UnsafeCell<List>, + /// Required by the `linked_list::Pointers` field. + _pin: PhantomPinned, +} + +generate_addr_of_methods! { + impl<T> ListEntry<T> { + unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<ListEntry<T>>> { + &self.pointers + } + } +} + +// With mutable access to the `IdleNotifiedSet`, you can get mutable access to +// the values. +unsafe impl<T: Send> Send for IdleNotifiedSet<T> {} +// With the current API we strictly speaking don't even need `T: Sync`, but we +// require it anyway to support adding &self APIs that access the values in the +// future. +unsafe impl<T: Sync> Sync for IdleNotifiedSet<T> {} + +// These impls control when it is safe to create a Waker. Since the waker does +// not allow access to the value in any way (including its destructor), it is +// not necessary for `T` to be Send or Sync. +unsafe impl<T> Send for ListEntry<T> {} +unsafe impl<T> Sync for ListEntry<T> {} + +impl<T> IdleNotifiedSet<T> { + /// Create a new IdleNotifiedSet. + pub(crate) fn new() -> Self { + let lists = Mutex::new(ListsInner { + notified: LinkedList::new(), + idle: LinkedList::new(), + waker: None, + }); + + IdleNotifiedSet { + lists: Arc::new(lists), + length: 0, + } + } + + pub(crate) fn len(&self) -> usize { + self.length + } + + pub(crate) fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Insert the given value into the `idle` list. + pub(crate) fn insert_idle(&mut self, value: T) -> EntryInOneOfTheLists<'_, T> { + self.length += 1; + + let entry = Arc::new(ListEntry { + parent: self.lists.clone(), + value: UnsafeCell::new(ManuallyDrop::new(value)), + my_list: UnsafeCell::new(List::Idle), + pointers: linked_list::Pointers::new(), + _pin: PhantomPinned, + }); + + { + let mut lock = self.lists.lock(); + lock.idle.push_front(entry.clone()); + } + + // Safety: We just put the entry in the idle list, so it is in one of the lists. + EntryInOneOfTheLists { entry, set: self } + } + + /// Pop an entry from the notified list to poll it. The entry is moved to + /// the idle list atomically. + pub(crate) fn pop_notified(&mut self, waker: &Waker) -> Option<EntryInOneOfTheLists<'_, T>> { + // We don't decrement the length because this call moves the entry to + // the idle list rather than removing it. + if self.length == 0 { + // Fast path. + return None; + } + + let mut lock = self.lists.lock(); + + let should_update_waker = match lock.waker.as_mut() { + Some(cur_waker) => !waker.will_wake(cur_waker), + None => true, + }; + if should_update_waker { + lock.waker = Some(waker.clone()); + } + + // Pop the entry, returning None if empty. + let entry = lock.notified.pop_back()?; + + lock.idle.push_front(entry.clone()); + + // Safety: We are holding the lock. + entry.my_list.with_mut(|ptr| unsafe { + *ptr = List::Idle; + }); + + drop(lock); + + // Safety: We just put the entry in the idle list, so it is in one of the lists. + Some(EntryInOneOfTheLists { entry, set: self }) + } + + /// Call a function on every element in this list. + pub(crate) fn for_each<F: FnMut(&mut T)>(&mut self, mut func: F) { + fn get_ptrs<T>(list: &mut LinkedList<T>, ptrs: &mut Vec<*mut T>) { + let mut node = list.last(); + + while let Some(entry) = node { + ptrs.push(entry.value.with_mut(|ptr| { + let ptr: *mut ManuallyDrop<T> = ptr; + let ptr: *mut T = ptr.cast(); + ptr + })); + + let prev = entry.pointers.get_prev(); + node = prev.map(|prev| unsafe { &*prev.as_ptr() }); + } + } + + // Atomically get a raw pointer to the value of every entry. + // + // Since this only locks the mutex once, it is not possible for a value + // to get moved from the idle list to the notified list during the + // operation, which would otherwise result in some value being listed + // twice. + let mut ptrs = Vec::with_capacity(self.len()); + { + let mut lock = self.lists.lock(); + + get_ptrs(&mut lock.idle, &mut ptrs); + get_ptrs(&mut lock.notified, &mut ptrs); + } + debug_assert_eq!(ptrs.len(), ptrs.capacity()); + + for ptr in ptrs { + // Safety: When we grabbed the pointers, the entries were in one of + // the two lists. This means that their value was valid at the time, + // and it must still be valid because we are the IdleNotifiedSet, + // and only we can remove an entry from the two lists. (It's + // possible that an entry is moved from one list to the other during + // this loop, but that is ok.) + func(unsafe { &mut *ptr }); + } + } + + /// Remove all entries in both lists, applying some function to each element. + /// + /// The closure is called on all elements even if it panics. Having it panic + /// twice is a double-panic, and will abort the application. + pub(crate) fn drain<F: FnMut(T)>(&mut self, func: F) { + if self.length == 0 { + // Fast path. + return; + } + self.length = 0; + + // The LinkedList is not cleared on panic, so we use a bomb to clear it. + // + // This value has the invariant that any entry in its `all_entries` list + // has `my_list` set to `Neither` and that the value has not yet been + // dropped. + struct AllEntries<T, F: FnMut(T)> { + all_entries: LinkedList<T>, + func: F, + } + + impl<T, F: FnMut(T)> AllEntries<T, F> { + fn pop_next(&mut self) -> bool { + if let Some(entry) = self.all_entries.pop_back() { + // Safety: We just took this value from the list, so we can + // destroy the value in the entry. + entry + .value + .with_mut(|ptr| unsafe { (self.func)(ManuallyDrop::take(&mut *ptr)) }); + true + } else { + false + } + } + } + + impl<T, F: FnMut(T)> Drop for AllEntries<T, F> { + fn drop(&mut self) { + while self.pop_next() {} + } + } + + let mut all_entries = AllEntries { + all_entries: LinkedList::new(), + func, + }; + + // Atomically move all entries to the new linked list in the AllEntries + // object. + { + let mut lock = self.lists.lock(); + unsafe { + // Safety: We are holding the lock and `all_entries` is a new + // LinkedList. + move_to_new_list(&mut lock.idle, &mut all_entries.all_entries); + move_to_new_list(&mut lock.notified, &mut all_entries.all_entries); + } + } + + // Keep destroying entries in the list until it is empty. + // + // If the closure panics, then the destructor of the `AllEntries` bomb + // ensures that we keep running the destructor on the remaining values. + // A second panic will abort the program. + while all_entries.pop_next() {} + } +} + +/// # Safety +/// +/// The mutex for the entries must be held, and the target list must be such +/// that setting `my_list` to `Neither` is ok. +unsafe fn move_to_new_list<T>(from: &mut LinkedList<T>, to: &mut LinkedList<T>) { + while let Some(entry) = from.pop_back() { + entry.my_list.with_mut(|ptr| { + *ptr = List::Neither; + }); + to.push_front(entry); + } +} + +impl<'a, T> EntryInOneOfTheLists<'a, T> { + /// Remove this entry from the list it is in, returning the value associated + /// with the entry. + /// + /// This consumes the value, since it is no longer guaranteed to be in a + /// list. + pub(crate) fn remove(self) -> T { + self.set.length -= 1; + + { + let mut lock = self.set.lists.lock(); + + // Safety: We are holding the lock so there is no race, and we will + // remove the entry afterwards to uphold invariants. + let old_my_list = self.entry.my_list.with_mut(|ptr| unsafe { + let old_my_list = *ptr; + *ptr = List::Neither; + old_my_list + }); + + let list = match old_my_list { + List::Idle => &mut lock.idle, + List::Notified => &mut lock.notified, + // An entry in one of the lists is in one of the lists. + List::Neither => unreachable!(), + }; + + unsafe { + // Safety: We just checked that the entry is in this particular + // list. + list.remove(ListEntry::as_raw(&self.entry)).unwrap(); + } + } + + // By setting `my_list` to `Neither`, we have taken ownership of the + // value. We return it to the caller. + // + // Safety: We have a mutable reference to the `IdleNotifiedSet` that + // owns this entry, so we can use its permission to access the value. + self.entry + .value + .with_mut(|ptr| unsafe { ManuallyDrop::take(&mut *ptr) }) + } + + /// Access the value in this entry together with a context for its waker. + pub(crate) fn with_value_and_context<F, U>(&mut self, func: F) -> U + where + F: FnOnce(&mut T, &mut Context<'_>) -> U, + T: 'static, + { + let waker = waker_ref(&self.entry); + + let mut context = Context::from_waker(&waker); + + // Safety: We have a mutable reference to the `IdleNotifiedSet` that + // owns this entry, so we can use its permission to access the value. + self.entry + .value + .with_mut(|ptr| unsafe { func(&mut *ptr, &mut context) }) + } +} + +impl<T> Drop for IdleNotifiedSet<T> { + fn drop(&mut self) { + // Clear both lists. + self.drain(drop); + + #[cfg(debug_assertions)] + if !std::thread::panicking() { + let lock = self.lists.lock(); + assert!(lock.idle.is_empty()); + assert!(lock.notified.is_empty()); + } + } +} + +impl<T: 'static> Wake for ListEntry<T> { + fn wake_by_ref(me: &Arc<Self>) { + let mut lock = me.parent.lock(); + + // Safety: We are holding the lock and we will update the lists to + // maintain invariants. + let old_my_list = me.my_list.with_mut(|ptr| unsafe { + let old_my_list = *ptr; + if old_my_list == List::Idle { + *ptr = List::Notified; + } + old_my_list + }); + + if old_my_list == List::Idle { + // We move ourself to the notified list. + let me = unsafe { + // Safety: We just checked that we are in this particular list. + lock.idle.remove(NonNull::from(&**me)).unwrap() + }; + lock.notified.push_front(me); + + if let Some(waker) = lock.waker.take() { + drop(lock); + waker.wake(); + } + } + } + + fn wake(me: Arc<Self>) { + Self::wake_by_ref(&me) + } +} + +/// # Safety +/// +/// `ListEntry` is forced to be !Unpin. +unsafe impl<T> linked_list::Link for ListEntry<T> { + type Handle = Arc<ListEntry<T>>; + type Target = ListEntry<T>; + + fn as_raw(handle: &Self::Handle) -> NonNull<ListEntry<T>> { + let ptr: *const ListEntry<T> = Arc::as_ptr(handle); + // Safety: We can't get a null pointer from `Arc::as_ptr`. + unsafe { NonNull::new_unchecked(ptr as *mut ListEntry<T>) } + } + + unsafe fn from_raw(ptr: NonNull<ListEntry<T>>) -> Arc<ListEntry<T>> { + Arc::from_raw(ptr.as_ptr()) + } + + unsafe fn pointers( + target: NonNull<ListEntry<T>>, + ) -> NonNull<linked_list::Pointers<ListEntry<T>>> { + ListEntry::addr_of_pointers(target) + } +} diff --git a/src/util/linked_list.rs b/src/util/linked_list.rs index 894d216..b46bd6d 100644 --- a/src/util/linked_list.rs +++ b/src/util/linked_list.rs @@ -57,6 +57,13 @@ pub(crate) unsafe trait Link { unsafe fn from_raw(ptr: NonNull<Self::Target>) -> Self::Handle; /// Return the pointers for a node + /// + /// # Safety + /// + /// The resulting pointer should have the same tag in the stacked-borrows + /// stack as the argument. In particular, the method may not create an + /// intermediate reference in the process of creating the resulting raw + /// pointer. unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>>; } @@ -119,7 +126,7 @@ impl<L: Link> LinkedList<L, L::Target> { pub(crate) fn push_front(&mut self, val: L::Handle) { // The value should not be dropped, it is being inserted into the list let val = ManuallyDrop::new(val); - let ptr = L::as_raw(&*val); + let ptr = L::as_raw(&val); assert_ne!(self.head, Some(ptr)); unsafe { L::pointers(ptr).as_mut().set_next(self.head); @@ -219,6 +226,7 @@ impl<L: Link> fmt::Debug for LinkedList<L, L::Target> { #[cfg(any( feature = "fs", + feature = "rt", all(unix, feature = "process"), feature = "signal", feature = "sync", @@ -296,7 +304,7 @@ impl<T> Pointers<T> { } } - fn get_prev(&self) -> Option<NonNull<T>> { + pub(crate) fn get_prev(&self) -> Option<NonNull<T>> { // SAFETY: prev is the first field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -304,7 +312,7 @@ impl<T> Pointers<T> { ptr::read(prev) } } - fn get_next(&self) -> Option<NonNull<T>> { + pub(crate) fn get_next(&self) -> Option<NonNull<T>> { // SAFETY: next is the second field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -352,6 +360,7 @@ mod tests { use std::pin::Pin; #[derive(Debug)] + #[repr(C)] struct Entry { pointers: Pointers<Entry>, val: i32, @@ -369,8 +378,8 @@ mod tests { Pin::new_unchecked(&*ptr.as_ptr()) } - unsafe fn pointers(mut target: NonNull<Entry>) -> NonNull<Pointers<Entry>> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull<Entry>) -> NonNull<Pointers<Entry>> { + target.cast() } } @@ -614,6 +623,7 @@ mod tests { } } + #[cfg(not(tokio_wasm))] proptest::proptest! { #[test] fn fuzz_linked_list(ops: Vec<usize>) { @@ -621,6 +631,7 @@ mod tests { } } + #[cfg(not(tokio_wasm))] fn run_fuzz(ops: Vec<usize>) { use std::collections::VecDeque; diff --git a/src/util/mod.rs b/src/util/mod.rs index df30f2b..9f6119a 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -3,6 +3,17 @@ cfg_io_driver! { pub(crate) mod slab; } +#[cfg(feature = "rt")] +pub(crate) mod atomic_cell; + +#[cfg(any( + feature = "rt", + feature = "signal", + feature = "process", + tokio_no_const_mutex_new, +))] +pub(crate) mod once_cell; + #[cfg(any( // io driver uses `WakeList` directly feature = "net", @@ -37,10 +48,15 @@ pub(crate) use wake_list::WakeList; ))] pub(crate) mod linked_list; -#[cfg(any(feature = "rt-multi-thread", feature = "macros"))] -mod rand; +#[cfg(any(feature = "rt", feature = "macros"))] +pub(crate) mod rand; cfg_rt! { + mod idle_notified_set; + pub(crate) use idle_notified_set::IdleNotifiedSet; + + pub(crate) use self::rand::RngSeedGenerator; + mod wake; pub(crate) use wake::WakerRef; pub(crate) use wake::{waker_ref, Wake}; @@ -48,28 +64,15 @@ cfg_rt! { mod sync_wrapper; pub(crate) use sync_wrapper::SyncWrapper; - mod vec_deque_cell; - pub(crate) use vec_deque_cell::VecDequeCell; + mod rc_cell; + pub(crate) use rc_cell::RcCell; } cfg_rt_multi_thread! { - pub(crate) use self::rand::FastRand; - mod try_lock; pub(crate) use try_lock::TryLock; } pub(crate) mod trace; -#[cfg(any(feature = "macros"))] -#[cfg_attr(not(feature = "macros"), allow(unreachable_pub))] -pub use self::rand::thread_rng_n; - -#[cfg(any( - feature = "rt", - feature = "time", - feature = "net", - feature = "process", - all(unix, feature = "signal") -))] pub(crate) mod error; diff --git a/src/util/once_cell.rs b/src/util/once_cell.rs new file mode 100644 index 0000000..1925f0a --- /dev/null +++ b/src/util/once_cell.rs @@ -0,0 +1,70 @@ +#![allow(dead_code)] +use std::cell::UnsafeCell; +use std::mem::MaybeUninit; +use std::sync::Once; + +pub(crate) struct OnceCell<T> { + once: Once, + value: UnsafeCell<MaybeUninit<T>>, +} + +unsafe impl<T: Send + Sync> Send for OnceCell<T> {} +unsafe impl<T: Send + Sync> Sync for OnceCell<T> {} + +impl<T> OnceCell<T> { + pub(crate) const fn new() -> Self { + Self { + once: Once::new(), + value: UnsafeCell::new(MaybeUninit::uninit()), + } + } + + /// Get the value inside this cell, intiailizing it using the provided + /// function if necessary. + /// + /// If the `init` closure panics, then the `OnceCell` is poisoned and all + /// future calls to `get` will panic. + #[inline] + pub(crate) fn get(&self, init: impl FnOnce() -> T) -> &T { + if !self.once.is_completed() { + self.do_init(init); + } + + // Safety: The `std::sync::Once` guarantees that we can only reach this + // line if a `call_once` closure has been run exactly once and without + // panicking. Thus, the value is not uninitialized. + // + // There is also no race because the only `&self` method that modifies + // `value` is `do_init`, but if the `call_once` closure is still + // running, then no thread has gotten past the `call_once`. + unsafe { &*(self.value.get() as *const T) } + } + + #[cold] + fn do_init(&self, init: impl FnOnce() -> T) { + let value_ptr = self.value.get() as *mut T; + + self.once.call_once(|| { + let set_to = init(); + + // Safety: The `std::sync::Once` guarantees that this initialization + // will run at most once, and that no thread can get past the + // `call_once` until it has run exactly once. Thus, we have + // exclusive access to `value`. + unsafe { + std::ptr::write(value_ptr, set_to); + } + }); + } +} + +impl<T> Drop for OnceCell<T> { + fn drop(&mut self) { + if self.once.is_completed() { + let value_ptr = self.value.get() as *mut T; + unsafe { + std::ptr::drop_in_place(value_ptr); + } + } + } +} diff --git a/src/util/rand.rs b/src/util/rand.rs index 6b19c8b..749da6b 100644 --- a/src/util/rand.rs +++ b/src/util/rand.rs @@ -1,5 +1,103 @@ use std::cell::Cell; +cfg_rt! { + use std::sync::Mutex; + + /// A deterministic generator for seeds (and other generators). + /// + /// Given the same initial seed, the generator will output the same sequence of seeds. + /// + /// Since the seed generator will be kept in a runtime handle, we need to wrap `FastRand` + /// in a Mutex to make it thread safe. Different to the `FastRand` that we keep in a + /// thread local store, the expectation is that seed generation will not need to happen + /// very frequently, so the cost of the mutex should be minimal. + #[derive(Debug)] + pub(crate) struct RngSeedGenerator { + /// Internal state for the seed generator. We keep it in a Mutex so that we can safely + /// use it across multiple threads. + state: Mutex<FastRand>, + } + + impl RngSeedGenerator { + /// Returns a new generator from the provided seed. + pub(crate) fn new(seed: RngSeed) -> Self { + Self { + state: Mutex::new(FastRand::new(seed)), + } + } + + /// Returns the next seed in the sequence. + pub(crate) fn next_seed(&self) -> RngSeed { + let rng = self + .state + .lock() + .expect("RNG seed generator is internally corrupt"); + + let s = rng.fastrand(); + let r = rng.fastrand(); + + RngSeed::from_pair(s, r) + } + + /// Directly creates a generator using the next seed. + pub(crate) fn next_generator(&self) -> Self { + RngSeedGenerator::new(self.next_seed()) + } + } +} + +/// A seed for random number generation. +/// +/// In order to make certain functions within a runtime deterministic, a seed +/// can be specified at the time of creation. +#[allow(unreachable_pub)] +#[derive(Clone, Debug)] +pub struct RngSeed { + s: u32, + r: u32, +} + +impl RngSeed { + /// Creates a random seed using loom internally. + pub(crate) fn new() -> Self { + Self::from_u64(crate::loom::rand::seed()) + } + + cfg_unstable! { + /// Generates a seed from the provided byte slice. + /// + /// # Example + /// + /// ``` + /// # use tokio::runtime::RngSeed; + /// let seed = RngSeed::from_bytes(b"make me a seed"); + /// ``` + #[cfg(feature = "rt")] + pub fn from_bytes(bytes: &[u8]) -> Self { + use std::{collections::hash_map::DefaultHasher, hash::Hasher}; + + let mut hasher = DefaultHasher::default(); + hasher.write(bytes); + Self::from_u64(hasher.finish()) + } + } + + fn from_u64(seed: u64) -> Self { + let one = (seed >> 32) as u32; + let mut two = seed as u32; + + if two == 0 { + // This value cannot be zero + two = 1; + } + + Self::from_pair(one, two) + } + + fn from_pair(s: u32, r: u32) -> Self { + Self { s, r } + } +} /// Fast random number generate. /// /// Implement xorshift64+: 2 32-bit xorshift sequences added together. @@ -15,21 +113,29 @@ pub(crate) struct FastRand { impl FastRand { /// Initializes a new, thread-local, fast random number generator. - pub(crate) fn new(seed: u64) -> FastRand { - let one = (seed >> 32) as u32; - let mut two = seed as u32; - - if two == 0 { - // This value cannot be zero - two = 1; - } - + pub(crate) fn new(seed: RngSeed) -> FastRand { FastRand { - one: Cell::new(one), - two: Cell::new(two), + one: Cell::new(seed.s), + two: Cell::new(seed.r), } } + /// Replaces the state of the random number generator with the provided seed, returning + /// the seed that represents the previous state of the random number generator. + /// + /// The random number generator will become equivalent to one created with + /// the same seed. + #[cfg(feature = "rt")] + pub(crate) fn replace_seed(&self, seed: RngSeed) -> RngSeed { + let old_seed = RngSeed::from_pair(self.one.get(), self.two.get()); + + self.one.replace(seed.s); + self.two.replace(seed.r); + + old_seed + } + + #[cfg(any(feature = "macros", feature = "rt-multi-thread"))] pub(crate) fn fastrand_n(&self, n: u32) -> u32 { // This is similar to fastrand() % n, but faster. // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ @@ -50,15 +156,3 @@ impl FastRand { s0.wrapping_add(s1) } } - -// Used by the select macro and `StreamMap` -#[cfg(any(feature = "macros"))] -#[doc(hidden)] -#[cfg_attr(not(feature = "macros"), allow(unreachable_pub))] -pub fn thread_rng_n(n: u32) -> u32 { - thread_local! { - static THREAD_RNG: FastRand = FastRand::new(crate::loom::rand::seed()); - } - - THREAD_RNG.with(|rng| rng.fastrand_n(n)) -} diff --git a/src/util/rc_cell.rs b/src/util/rc_cell.rs new file mode 100644 index 0000000..4472492 --- /dev/null +++ b/src/util/rc_cell.rs @@ -0,0 +1,57 @@ +use crate::loom::cell::UnsafeCell; + +use std::rc::Rc; + +/// This is exactly like `Cell<Option<Rc<T>>>`, except that it provides a `get` +/// method even though `Rc` is not `Copy`. +pub(crate) struct RcCell<T> { + inner: UnsafeCell<Option<Rc<T>>>, +} + +impl<T> RcCell<T> { + #[cfg(not(all(loom, test)))] + pub(crate) const fn new() -> Self { + Self { + inner: UnsafeCell::new(None), + } + } + + // The UnsafeCell in loom does not have a const `new` fn. + #[cfg(all(loom, test))] + pub(crate) fn new() -> Self { + Self { + inner: UnsafeCell::new(None), + } + } + + /// Safety: This method may not be called recursively. + #[inline] + unsafe fn with_inner<F, R>(&self, f: F) -> R + where + F: FnOnce(&mut Option<Rc<T>>) -> R, + { + // safety: This type is not Sync, so concurrent calls of this method + // cannot happen. Furthermore, the caller guarantees that the method is + // not called recursively. Finally, this is the only place that can + // create mutable references to the inner Rc. This ensures that any + // mutable references created here are exclusive. + self.inner.with_mut(|ptr| f(&mut *ptr)) + } + + pub(crate) fn get(&self) -> Option<Rc<T>> { + // safety: The `Rc::clone` method will not call any unknown user-code, + // so it will not result in a recursive call to `with_inner`. + unsafe { self.with_inner(|rc| rc.clone()) } + } + + pub(crate) fn replace(&self, val: Option<Rc<T>>) -> Option<Rc<T>> { + // safety: No destructors or other unknown user-code will run inside the + // `with_inner` call, so no recursive call to `with_inner` can happen. + unsafe { self.with_inner(|rc| std::mem::replace(rc, val)) } + } + + pub(crate) fn set(&self, val: Option<Rc<T>>) { + let old = self.replace(val); + drop(old); + } +} diff --git a/src/util/slab.rs b/src/util/slab.rs index 97355d5..0e16e40 100644 --- a/src/util/slab.rs +++ b/src/util/slab.rs @@ -157,6 +157,12 @@ struct Slot<T> { /// Next entry in the free list. next: u32, + + /// Makes miri happy by making mutable references not take exclusive access. + /// + /// Could probably also be fixed by replacing `slots` with a raw-pointer + /// based equivalent. + _pin: std::marker::PhantomPinned, } /// Value paired with a reference to the page. @@ -409,7 +415,7 @@ impl<T: Entry> Page<T> { slot.value.with(|ptr| unsafe { (*ptr).value.reset() }); // Return a reference to the slot - Some((me.addr(idx), slot.gen_ref(me))) + Some((me.addr(idx), locked.gen_ref(idx, me))) } else if me.len == locked.slots.len() { // The page is full None @@ -428,9 +434,10 @@ impl<T: Entry> Page<T> { locked.slots.push(Slot { value: UnsafeCell::new(Value { value: Default::default(), - page: &**me as *const _, + page: Arc::as_ptr(me), }), next: 0, + _pin: std::marker::PhantomPinned, }); // Increment the head to indicate the free stack is empty @@ -443,7 +450,7 @@ impl<T: Entry> Page<T> { debug_assert_eq!(locked.slots.len(), locked.head); - Some((me.addr(idx), locked.slots[idx].gen_ref(me))) + Some((me.addr(idx), locked.gen_ref(idx, me))) } } } @@ -544,10 +551,9 @@ impl<T> Slots<T> { fn index_for(&self, slot: *const Value<T>) -> usize { use std::mem; - let base = &self.slots[0] as *const _ as usize; - - assert!(base != 0, "page is unallocated"); + assert_ne!(self.slots.capacity(), 0, "page is unallocated"); + let base = self.slots.as_ptr() as usize; let slot = slot as usize; let width = mem::size_of::<Slot<T>>(); @@ -558,18 +564,15 @@ impl<T> Slots<T> { idx } -} -impl<T: Entry> Slot<T> { - /// Generates a `Ref` for the slot. This involves bumping the page's ref count. - fn gen_ref(&self, page: &Arc<Page<T>>) -> Ref<T> { - // The ref holds a ref on the page. The `Arc` is forgotten here and is - // resurrected in `release` when the `Ref` is dropped. By avoiding to - // hold on to an explicit `Arc` value, the struct size of `Ref` is - // reduced. + /// Generates a `Ref` for the slot at the given index. This involves bumping the page's ref count. + fn gen_ref(&self, idx: usize, page: &Arc<Page<T>>) -> Ref<T> { + assert!(idx < self.slots.len()); mem::forget(page.clone()); - let slot = self as *const Slot<T>; - let value = slot as *const Value<T>; + + let vec_ptr = self.slots.as_ptr(); + let slot: *const Slot<T> = unsafe { vec_ptr.add(idx) }; + let value: *const Value<T> = slot as *const Value<T>; Ref { value } } @@ -691,11 +694,13 @@ mod test { #[test] fn insert_many() { + const MANY: usize = normal_or_miri(10_000, 50); + let mut slab = Slab::<Foo>::new(); let alloc = slab.allocator(); let mut entries = vec![]; - for i in 0..10_000 { + for i in 0..MANY { let (addr, val) = alloc.allocate().unwrap(); val.id.store(i, SeqCst); entries.push((addr, val)); @@ -708,15 +713,15 @@ mod test { entries.clear(); - for i in 0..10_000 { + for i in 0..MANY { let (addr, val) = alloc.allocate().unwrap(); - val.id.store(10_000 - i, SeqCst); + val.id.store(MANY - i, SeqCst); entries.push((addr, val)); } for (i, (addr, v)) in entries.iter().enumerate() { - assert_eq!(10_000 - i, v.id.load(SeqCst)); - assert_eq!(10_000 - i, slab.get(*addr).unwrap().id.load(SeqCst)); + assert_eq!(MANY - i, v.id.load(SeqCst)); + assert_eq!(MANY - i, slab.get(*addr).unwrap().id.load(SeqCst)); } } @@ -726,7 +731,7 @@ mod test { let alloc = slab.allocator(); let mut entries = vec![]; - for i in 0..10_000 { + for i in 0..normal_or_miri(10_000, 100) { let (addr, val) = alloc.allocate().unwrap(); val.id.store(i, SeqCst); entries.push((addr, val)); @@ -734,7 +739,7 @@ mod test { for _ in 0..10 { // Drop 1000 in reverse - for _ in 0..1_000 { + for _ in 0..normal_or_miri(1_000, 10) { entries.pop(); } @@ -753,7 +758,7 @@ mod test { let mut entries1 = vec![]; let mut entries2 = vec![]; - for i in 0..10_000 { + for i in 0..normal_or_miri(10_000, 100) { let (addr, val) = alloc.allocate().unwrap(); val.id.store(i, SeqCst); @@ -771,6 +776,14 @@ mod test { } } + const fn normal_or_miri(normal: usize, miri: usize) -> usize { + if cfg!(miri) { + miri + } else { + normal + } + } + #[test] fn compact_all() { let mut slab = Slab::<Foo>::new(); @@ -780,7 +793,7 @@ mod test { for _ in 0..2 { entries.clear(); - for i in 0..10_000 { + for i in 0..normal_or_miri(10_000, 100) { let (addr, val) = alloc.allocate().unwrap(); val.id.store(i, SeqCst); @@ -808,7 +821,7 @@ mod test { let alloc = slab.allocator(); let mut entries = vec![]; - for _ in 0..5 { + for _ in 0..normal_or_miri(5, 2) { entries.clear(); // Allocate a few pages + 1 diff --git a/src/util/trace.rs b/src/util/trace.rs index e3c26f9..76e8a6c 100644 --- a/src/util/trace.rs +++ b/src/util/trace.rs @@ -1,40 +1,90 @@ cfg_trace! { cfg_rt! { + use core::{ + pin::Pin, + task::{Context, Poll}, + }; + use pin_project_lite::pin_project; + use std::future::Future; pub(crate) use tracing::instrument::Instrumented; #[inline] - #[cfg_attr(tokio_track_caller, track_caller)] - pub(crate) fn task<F>(task: F, kind: &'static str, name: Option<&str>) -> Instrumented<F> { + #[track_caller] + pub(crate) fn task<F>(task: F, kind: &'static str, name: Option<&str>, id: u64) -> Instrumented<F> { use tracing::instrument::Instrument; - #[cfg(tokio_track_caller)] let location = std::panic::Location::caller(); - #[cfg(tokio_track_caller)] let span = tracing::trace_span!( target: "tokio::task", "runtime.spawn", %kind, task.name = %name.unwrap_or_default(), + task.id = id, loc.file = location.file(), loc.line = location.line(), loc.col = location.column(), ); - #[cfg(not(tokio_track_caller))] - let span = tracing::trace_span!( - target: "tokio::task", - "runtime.spawn", - %kind, - task.name = %name.unwrap_or_default(), - ); task.instrument(span) } + + pub(crate) fn async_op<P,F>(inner: P, resource_span: tracing::Span, source: &str, poll_op_name: &'static str, inherits_child_attrs: bool) -> InstrumentedAsyncOp<F> + where P: FnOnce() -> F { + resource_span.in_scope(|| { + let async_op_span = tracing::trace_span!("runtime.resource.async_op", source = source, inherits_child_attrs = inherits_child_attrs); + let enter = async_op_span.enter(); + let async_op_poll_span = tracing::trace_span!("runtime.resource.async_op.poll"); + let inner = inner(); + drop(enter); + let tracing_ctx = AsyncOpTracingCtx { + async_op_span, + async_op_poll_span, + resource_span: resource_span.clone(), + }; + InstrumentedAsyncOp { + inner, + tracing_ctx, + poll_op_name, + } + }) + } + + #[derive(Debug, Clone)] + pub(crate) struct AsyncOpTracingCtx { + pub(crate) async_op_span: tracing::Span, + pub(crate) async_op_poll_span: tracing::Span, + pub(crate) resource_span: tracing::Span, + } + + + pin_project! { + #[derive(Debug, Clone)] + pub(crate) struct InstrumentedAsyncOp<F> { + #[pin] + pub(crate) inner: F, + pub(crate) tracing_ctx: AsyncOpTracingCtx, + pub(crate) poll_op_name: &'static str + } + } + + impl<F: Future> Future for InstrumentedAsyncOp<F> { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + let this = self.project(); + let poll_op_name = &*this.poll_op_name; + let _res_enter = this.tracing_ctx.resource_span.enter(); + let _async_op_enter = this.tracing_ctx.async_op_span.enter(); + let _async_op_poll_enter = this.tracing_ctx.async_op_poll_span.enter(); + trace_poll_op!(poll_op_name, this.inner.poll(cx)) + } + } } } cfg_time! { - #[cfg_attr(tokio_track_caller, track_caller)] + #[track_caller] pub(crate) fn caller_location() -> Option<&'static std::panic::Location<'static>> { - #[cfg(all(tokio_track_caller, tokio_unstable, feature = "tracing"))] + #[cfg(all(tokio_unstable, feature = "tracing"))] return Some(std::panic::Location::caller()); - #[cfg(not(all(tokio_track_caller, tokio_unstable, feature = "tracing")))] + #[cfg(not(all(tokio_unstable, feature = "tracing")))] None } } @@ -42,7 +92,7 @@ cfg_time! { cfg_not_trace! { cfg_rt! { #[inline] - pub(crate) fn task<F>(task: F, _: &'static str, _name: Option<&str>) -> F { + pub(crate) fn task<F>(task: F, _: &'static str, _name: Option<&str>, _: u64) -> F { // nop task } diff --git a/src/util/vec_deque_cell.rs b/src/util/vec_deque_cell.rs deleted file mode 100644 index b4e124c..0000000 --- a/src/util/vec_deque_cell.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::loom::cell::UnsafeCell; - -use std::collections::VecDeque; -use std::marker::PhantomData; - -/// This type is like VecDeque, except that it is not Sync and can be modified -/// through immutable references. -pub(crate) struct VecDequeCell<T> { - inner: UnsafeCell<VecDeque<T>>, - _not_sync: PhantomData<*const ()>, -} - -// This is Send for the same reasons that RefCell<VecDeque<T>> is Send. -unsafe impl<T: Send> Send for VecDequeCell<T> {} - -impl<T> VecDequeCell<T> { - pub(crate) fn with_capacity(cap: usize) -> Self { - Self { - inner: UnsafeCell::new(VecDeque::with_capacity(cap)), - _not_sync: PhantomData, - } - } - - /// Safety: This method may not be called recursively. - #[inline] - unsafe fn with_inner<F, R>(&self, f: F) -> R - where - F: FnOnce(&mut VecDeque<T>) -> R, - { - // safety: This type is not Sync, so concurrent calls of this method - // cannot happen. Furthermore, the caller guarantees that the method is - // not called recursively. Finally, this is the only place that can - // create mutable references to the inner VecDeque. This ensures that - // any mutable references created here are exclusive. - self.inner.with_mut(|ptr| f(&mut *ptr)) - } - - pub(crate) fn pop_front(&self) -> Option<T> { - unsafe { self.with_inner(VecDeque::pop_front) } - } - - pub(crate) fn push_back(&self, item: T) { - unsafe { - self.with_inner(|inner| inner.push_back(item)); - } - } - - /// Replaces the inner VecDeque with an empty VecDeque and return the current - /// contents. - pub(crate) fn take(&self) -> VecDeque<T> { - unsafe { self.with_inner(|inner| std::mem::take(inner)) } - } -} diff --git a/src/util/wake.rs b/src/util/wake.rs index 8f89668..5526cbc 100644 --- a/src/util/wake.rs +++ b/src/util/wake.rs @@ -1,13 +1,14 @@ +use crate::loom::sync::Arc; + use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ops::Deref; -use std::sync::Arc; use std::task::{RawWaker, RawWakerVTable, Waker}; /// Simplified waking interface based on Arcs. -pub(crate) trait Wake: Send + Sync { +pub(crate) trait Wake: Send + Sync + Sized + 'static { /// Wake by value. - fn wake(self: Arc<Self>); + fn wake(arc_self: Arc<Self>); /// Wake by reference. fn wake_by_ref(arc_self: &Arc<Self>); @@ -30,7 +31,7 @@ impl Deref for WakerRef<'_> { /// Creates a reference to a `Waker` from a reference to `Arc<impl Wake>`. pub(crate) fn waker_ref<W: Wake>(wake: &Arc<W>) -> WakerRef<'_> { - let ptr = &**wake as *const _ as *const (); + let ptr = Arc::as_ptr(wake) as *const (); let waker = unsafe { Waker::from_raw(RawWaker::new(ptr, waker_vtable::<W>())) }; diff --git a/tests/_require_full.rs b/tests/_require_full.rs index 98455be..4b9698a 100644 --- a/tests/_require_full.rs +++ b/tests/_require_full.rs @@ -1,2 +1,8 @@ -#![cfg(not(feature = "full"))] +#[cfg(not(any(feature = "full", tokio_wasm)))] compile_error!("run main Tokio tests with `--features full`"); + +// CI sets `--cfg tokio_no_parking_lot` when trying to run tests with +// `parking_lot` disabled. This check prevents "silent failure" if `parking_lot` +// accidentally gets enabled. +#[cfg(all(tokio_no_parking_lot, feature = "parking_lot"))] +compile_error!("parking_lot feature enabled when it should not be"); diff --git a/tests/async_send_sync.rs b/tests/async_send_sync.rs index aa14970..e46d5c8 100644 --- a/tests/async_send_sync.rs +++ b/tests/async_send_sync.rs @@ -127,70 +127,111 @@ macro_rules! assert_value { }; } -assert_value!(tokio::fs::DirBuilder: Send & Sync & Unpin); -assert_value!(tokio::fs::DirEntry: Send & Sync & Unpin); -assert_value!(tokio::fs::File: Send & Sync & Unpin); -assert_value!(tokio::fs::OpenOptions: Send & Sync & Unpin); -assert_value!(tokio::fs::ReadDir: Send & Sync & Unpin); - -async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::metadata(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read_link(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::write(&str, Vec<u8>): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::DirBuilder::create(_, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::open(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::create(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync & !Unpin); +macro_rules! cfg_not_wasi { + ($($item:item)*) => { + $( + #[cfg(not(tokio_wasi))] + $item + )* + } +} + +// Manually re-implementation of `async_assert_fn` for `poll_fn`. The macro +// doesn't work for this particular case because constructing the closure +// is too complicated. +const _: fn() = || { + let pinned = std::marker::PhantomPinned; + let f = tokio::macros::support::poll_fn(move |_| { + // Use `pinned` to take ownership of it. + let _ = &pinned; + std::task::Poll::Pending::<()> + }); + require_send(&f); + require_sync(&f); + AmbiguousIfUnpin::some_item(&f); +}; + +cfg_not_wasi! { + mod fs { + use super::*; + assert_value!(tokio::fs::DirBuilder: Send & Sync & Unpin); + assert_value!(tokio::fs::DirEntry: Send & Sync & Unpin); + assert_value!(tokio::fs::File: Send & Sync & Unpin); + assert_value!(tokio::fs::OpenOptions: Send & Sync & Unpin); + assert_value!(tokio::fs::ReadDir: Send & Sync & Unpin); + + async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::metadata(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read_link(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::write(&str, Vec<u8>): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::DirBuilder::create(_, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::open(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::create(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync & !Unpin); + async_assert_fn!( + tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync & !Unpin + ); + } +} + +cfg_not_wasi! { + assert_value!(tokio::net::TcpSocket: Send & Sync & Unpin); + async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync & !Unpin); +} assert_value!(tokio::net::TcpListener: Send & Sync & Unpin); -assert_value!(tokio::net::TcpSocket: Send & Sync & Unpin); assert_value!(tokio::net::TcpStream: Send & Sync & Unpin); -assert_value!(tokio::net::UdpSocket: Send & Sync & Unpin); assert_value!(tokio::net::tcp::OwnedReadHalf: Send & Sync & Unpin); assert_value!(tokio::net::tcp::OwnedWriteHalf: Send & Sync & Unpin); assert_value!(tokio::net::tcp::ReadHalf<'_>: Send & Sync & Unpin); assert_value!(tokio::net::tcp::ReuniteError: Send & Sync & Unpin); assert_value!(tokio::net::tcp::WriteHalf<'_>: Send & Sync & Unpin); async_assert_fn!(tokio::net::TcpListener::accept(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::peek(_, &mut [u8]): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::readable(_): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::ready(_, tokio::io::Interest): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::writable(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::peek_from(_, &mut [u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::readable(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::ready(_, tokio::io::Interest): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::writable(_): Send & Sync & !Unpin); + +// Wasi does not support UDP +cfg_not_wasi! { + mod udp_socket { + use super::*; + assert_value!(tokio::net::UdpSocket: Send & Sync & Unpin); + async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::peek_from(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::readable(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::writable(_): Send & Sync & !Unpin); + } +} async_assert_fn!(tokio::net::lookup_host(SocketAddr): Send & Sync & !Unpin); async_assert_fn!(tokio::net::tcp::ReadHalf::peek(_, &mut [u8]): Send & Sync & !Unpin); @@ -242,16 +283,22 @@ mod windows_named_pipe { async_assert_fn!(NamedPipeServer::writable(_): Send & Sync & !Unpin); } -assert_value!(tokio::process::Child: Send & Sync & Unpin); -assert_value!(tokio::process::ChildStderr: Send & Sync & Unpin); -assert_value!(tokio::process::ChildStdin: Send & Sync & Unpin); -assert_value!(tokio::process::ChildStdout: Send & Sync & Unpin); -assert_value!(tokio::process::Command: Send & Sync & Unpin); -async_assert_fn!(tokio::process::Child::kill(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::process::Child::wait(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync & !Unpin); +cfg_not_wasi! { + mod test_process { + use super::*; + assert_value!(tokio::process::Child: Send & Sync & Unpin); + assert_value!(tokio::process::ChildStderr: Send & Sync & Unpin); + assert_value!(tokio::process::ChildStdin: Send & Sync & Unpin); + assert_value!(tokio::process::ChildStdout: Send & Sync & Unpin); + assert_value!(tokio::process::Command: Send & Sync & Unpin); + async_assert_fn!(tokio::process::Child::kill(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::process::Child::wait(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync & !Unpin); + } + + async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync & !Unpin); +} -async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync & !Unpin); #[cfg(unix)] mod unix_signal { use super::*; @@ -362,6 +409,14 @@ assert_value!(tokio::sync::watch::Ref<'_, YY>: !Send & Sync & Unpin); assert_value!(tokio::sync::watch::Sender<NN>: !Send & !Sync & Unpin); assert_value!(tokio::sync::watch::Sender<YN>: !Send & !Sync & Unpin); assert_value!(tokio::sync::watch::Sender<YY>: Send & Sync & Unpin); +assert_value!(tokio::task::JoinError: Send & Sync & Unpin); +assert_value!(tokio::task::JoinHandle<NN>: !Send & !Sync & Unpin); +assert_value!(tokio::task::JoinHandle<YN>: Send & Sync & Unpin); +assert_value!(tokio::task::JoinHandle<YY>: Send & Sync & Unpin); +assert_value!(tokio::task::JoinSet<NN>: !Send & !Sync & Unpin); +assert_value!(tokio::task::JoinSet<YN>: Send & Sync & Unpin); +assert_value!(tokio::task::JoinSet<YY>: Send & Sync & Unpin); +assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin); async_assert_fn!(tokio::sync::Barrier::wait(_): Send & Sync & !Unpin); async_assert_fn!(tokio::sync::Mutex<NN>::lock(_): !Send & !Sync & !Unpin); async_assert_fn!(tokio::sync::Mutex<NN>::lock_owned(_): !Send & !Sync & !Unpin); @@ -434,25 +489,25 @@ async_assert_fn!(tokio::sync::watch::Receiver<YY>::changed(_): Send & Sync & !Un async_assert_fn!(tokio::sync::watch::Sender<NN>::closed(_): !Send & !Sync & !Unpin); async_assert_fn!(tokio::sync::watch::Sender<YN>::closed(_): !Send & !Sync & !Unpin); async_assert_fn!(tokio::sync::watch::Sender<YY>::closed(_): Send & Sync & !Unpin); - -async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSync<()>): Send & Sync & !Unpin); -async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSend<()>): Send & !Sync & !Unpin); -async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFuture<()>): !Send & !Sync & !Unpin); -async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSync<()>): Send & !Sync & !Unpin); -async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSend<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet<Cell<u32>>::join_next(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet<Cell<u32>>::shutdown(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet<Rc<u32>>::join_next(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet<Rc<u32>>::shutdown(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet<u32>::join_next(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet<u32>::shutdown(_): Send & Sync & !Unpin); async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFuture<()>): !Send & !Sync & !Unpin); -async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSync<()>): !Send & !Sync & !Unpin); -async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSend<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSend<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSync<()>): Send & !Sync & !Unpin); async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFuture<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSend<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSync<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFuture<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSend<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSync<()>): Send & Sync & !Unpin); async_assert_fn!(tokio::task::LocalSet::run_until(_, BoxFutureSync<()>): !Send & !Sync & !Unpin); async_assert_fn!(tokio::task::unconstrained(BoxFuture<()>): !Send & !Sync & Unpin); async_assert_fn!(tokio::task::unconstrained(BoxFutureSend<()>): Send & !Sync & Unpin); async_assert_fn!(tokio::task::unconstrained(BoxFutureSync<()>): Send & Sync & Unpin); -assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin); -assert_value!(tokio::task::JoinHandle<YY>: Send & Sync & Unpin); -assert_value!(tokio::task::JoinHandle<YN>: Send & Sync & Unpin); -assert_value!(tokio::task::JoinHandle<NN>: !Send & !Sync & Unpin); -assert_value!(tokio::task::JoinError: Send & Sync & Unpin); assert_value!(tokio::runtime::Builder: Send & Sync & Unpin); assert_value!(tokio::runtime::EnterGuard<'_>: Send & Sync & Unpin); diff --git a/tests/buffered.rs b/tests/buffered.rs index 98b6d5f..4251c3f 100644 --- a/tests/buffered.rs +++ b/tests/buffered.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind() use tokio::net::TcpListener; use tokio_test::assert_ok; @@ -18,10 +18,10 @@ async fn echo_server() { let msg = "foo bar baz"; let t = thread::spawn(move || { - let mut s = assert_ok!(TcpStream::connect(&addr)); + let mut s = assert_ok!(TcpStream::connect(addr)); let t2 = thread::spawn(move || { - let mut s = assert_ok!(TcpStream::connect(&addr)); + let mut s = assert_ok!(TcpStream::connect(addr)); let mut b = vec![0; msg.len() * N]; assert_ok!(s.read_exact(&mut b)); b diff --git a/tests/fs.rs b/tests/fs.rs index 13c44c0..ba38b71 100644 --- a/tests/fs.rs +++ b/tests/fs.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations use tokio::fs; use tokio_test::assert_ok; diff --git a/tests/fs_copy.rs b/tests/fs_copy.rs index 8d16320..04678cf 100644 --- a/tests/fs_copy.rs +++ b/tests/fs_copy.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations use tempfile::tempdir; use tokio::fs; diff --git a/tests/fs_dir.rs b/tests/fs_dir.rs index 21efe8c..f197a40 100644 --- a/tests/fs_dir.rs +++ b/tests/fs_dir.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support directory operations use tokio::fs; use tokio_test::{assert_err, assert_ok}; diff --git a/tests/fs_file.rs b/tests/fs_file.rs index f645e61..603ccad 100644 --- a/tests/fs_file.rs +++ b/tests/fs_file.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations use std::io::prelude::*; use tempfile::NamedTempFile; @@ -73,7 +73,7 @@ async fn coop() { let mut buf = [0; 1024]; loop { - file.read(&mut buf).await.unwrap(); + let _ = file.read(&mut buf).await.unwrap(); file.seek(std::io::SeekFrom::Start(0)).await.unwrap(); } }); diff --git a/tests/fs_link.rs b/tests/fs_link.rs index 2ef666f..d198abc 100644 --- a/tests/fs_link.rs +++ b/tests/fs_link.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations use tokio::fs; diff --git a/tests/io_buf_writer.rs b/tests/io_buf_writer.rs index 47a0d46..d3acf62 100644 --- a/tests/io_buf_writer.rs +++ b/tests/io_buf_writer.rs @@ -70,15 +70,15 @@ where async fn buf_writer() { let mut writer = BufWriter::with_capacity(2, Vec::new()); - writer.write(&[0, 1]).await.unwrap(); + assert_eq!(writer.write(&[0, 1]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1]); - writer.write(&[2]).await.unwrap(); + assert_eq!(writer.write(&[2]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2]); assert_eq!(*writer.get_ref(), [0, 1]); - writer.write(&[3]).await.unwrap(); + assert_eq!(writer.write(&[3]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2, 3]); assert_eq!(*writer.get_ref(), [0, 1]); @@ -86,20 +86,20 @@ async fn buf_writer() { assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); - writer.write(&[4]).await.unwrap(); - writer.write(&[5]).await.unwrap(); + assert_eq!(writer.write(&[4]).await.unwrap(), 1); + assert_eq!(writer.write(&[5]).await.unwrap(), 1); assert_eq!(writer.buffer(), [4, 5]); assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); - writer.write(&[6]).await.unwrap(); + assert_eq!(writer.write(&[6]).await.unwrap(), 1); assert_eq!(writer.buffer(), [6]); assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5]); - writer.write(&[7, 8]).await.unwrap(); + assert_eq!(writer.write(&[7, 8]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8]); - writer.write(&[9, 10, 11]).await.unwrap(); + assert_eq!(writer.write(&[9, 10, 11]).await.unwrap(), 3); assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); @@ -111,7 +111,7 @@ async fn buf_writer() { #[tokio::test] async fn buf_writer_inner_flushes() { let mut w = BufWriter::with_capacity(3, Vec::new()); - w.write(&[0, 1]).await.unwrap(); + assert_eq!(w.write(&[0, 1]).await.unwrap(), 2); assert_eq!(*w.get_ref(), []); w.flush().await.unwrap(); let w = w.into_inner(); @@ -135,15 +135,15 @@ async fn buf_writer_seek() { async fn maybe_pending_buf_writer() { let mut writer = BufWriter::with_capacity(2, MaybePending::new(Vec::new())); - writer.write(&[0, 1]).await.unwrap(); + assert_eq!(writer.write(&[0, 1]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(&writer.get_ref().inner, &[0, 1]); - writer.write(&[2]).await.unwrap(); + assert_eq!(writer.write(&[2]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2]); assert_eq!(&writer.get_ref().inner, &[0, 1]); - writer.write(&[3]).await.unwrap(); + assert_eq!(writer.write(&[3]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2, 3]); assert_eq!(&writer.get_ref().inner, &[0, 1]); @@ -151,20 +151,20 @@ async fn maybe_pending_buf_writer() { assert_eq!(writer.buffer(), []); assert_eq!(&writer.get_ref().inner, &[0, 1, 2, 3]); - writer.write(&[4]).await.unwrap(); - writer.write(&[5]).await.unwrap(); + assert_eq!(writer.write(&[4]).await.unwrap(), 1); + assert_eq!(writer.write(&[5]).await.unwrap(), 1); assert_eq!(writer.buffer(), [4, 5]); assert_eq!(&writer.get_ref().inner, &[0, 1, 2, 3]); - writer.write(&[6]).await.unwrap(); + assert_eq!(writer.write(&[6]).await.unwrap(), 1); assert_eq!(writer.buffer(), [6]); assert_eq!(writer.get_ref().inner, &[0, 1, 2, 3, 4, 5]); - writer.write(&[7, 8]).await.unwrap(); + assert_eq!(writer.write(&[7, 8]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(writer.get_ref().inner, &[0, 1, 2, 3, 4, 5, 6, 7, 8]); - writer.write(&[9, 10, 11]).await.unwrap(); + assert_eq!(writer.write(&[9, 10, 11]).await.unwrap(), 3); assert_eq!(writer.buffer(), []); assert_eq!( writer.get_ref().inner, @@ -182,7 +182,7 @@ async fn maybe_pending_buf_writer() { #[tokio::test] async fn maybe_pending_buf_writer_inner_flushes() { let mut w = BufWriter::with_capacity(3, MaybePending::new(Vec::new())); - w.write(&[0, 1]).await.unwrap(); + assert_eq!(w.write(&[0, 1]).await.unwrap(), 2); assert_eq!(&w.get_ref().inner, &[]); w.flush().await.unwrap(); let w = w.into_inner().inner; diff --git a/tests/io_copy_bidirectional.rs b/tests/io_copy_bidirectional.rs index 0e82b29..c549675 100644 --- a/tests/io_copy_bidirectional.rs +++ b/tests/io_copy_bidirectional.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind() use std::time::Duration; use tokio::io::{self, copy_bidirectional, AsyncReadExt, AsyncWriteExt}; @@ -111,18 +111,30 @@ async fn blocking_one_side_does_not_block_other() { } #[tokio::test] -async fn immediate_exit_on_error() { - symmetric(|handle, mut a, mut b| async move { - block_write(&mut a).await; +async fn immediate_exit_on_write_error() { + let payload = b"here, take this"; + let error = || io::Error::new(io::ErrorKind::Other, "no thanks!"); - // Fill up the b->copy->a path. We expect that this will _not_ drain - // before we exit the copy task. - let _bytes_written = block_write(&mut b).await; + let mut a = tokio_test::io::Builder::new() + .read(payload) + .write_error(error()) + .build(); - // Drop b. We should not wait for a to consume the data buffered in the - // copy loop, since b will be failing writes. - drop(b); - assert!(handle.await.unwrap().is_err()); - }) - .await + let mut b = tokio_test::io::Builder::new() + .read(payload) + .write_error(error()) + .build(); + + assert!(copy_bidirectional(&mut a, &mut b).await.is_err()); +} + +#[tokio::test] +async fn immediate_exit_on_read_error() { + let error = || io::Error::new(io::ErrorKind::Other, "got nothing!"); + + let mut a = tokio_test::io::Builder::new().read_error(error()).build(); + + let mut b = tokio_test::io::Builder::new().read_error(error()).build(); + + assert!(copy_bidirectional(&mut a, &mut b).await.is_err()); } diff --git a/tests/io_driver.rs b/tests/io_driver.rs index 6fb566d..97018e0 100644 --- a/tests/io_driver.rs +++ b/tests/io_driver.rs @@ -1,5 +1,6 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +// Wasi does not support panic recovery or threading +#![cfg(all(feature = "full", not(tokio_wasi)))] use tokio::net::TcpListener; use tokio::runtime; @@ -79,7 +80,7 @@ fn test_drop_on_notify() { drop(task); // Establish a connection to the acceptor - let _s = TcpStream::connect(&addr).unwrap(); + let _s = TcpStream::connect(addr).unwrap(); // Force the reactor to turn rt.block_on(async {}); diff --git a/tests/io_driver_drop.rs b/tests/io_driver_drop.rs index 631e66e..0a384d4 100644 --- a/tests/io_driver_drop.rs +++ b/tests/io_driver_drop.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind use tokio::net::TcpListener; use tokio::runtime; diff --git a/tests/io_fill_buf.rs b/tests/io_fill_buf.rs index 0b2ebd7..62c3f1b 100644 --- a/tests/io_fill_buf.rs +++ b/tests/io_fill_buf.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations use tempfile::NamedTempFile; use tokio::fs::File; diff --git a/tests/io_mem_stream.rs b/tests/io_mem_stream.rs index 520391a..3d9941d 100644 --- a/tests/io_mem_stream.rs +++ b/tests/io_mem_stream.rs @@ -101,3 +101,22 @@ async fn max_write_size() { // drop b only after task t1 finishes writing drop(b); } + +#[tokio::test] +async fn duplex_is_cooperative() { + let (mut tx, mut rx) = tokio::io::duplex(1024 * 8); + + tokio::select! { + biased; + + _ = async { + loop { + let buf = [3u8; 4096]; + tx.write_all(&buf).await.unwrap(); + let mut buf = [0u8; 4096]; + let _ = rx.read(&mut buf).await.unwrap(); + } + } => {}, + _ = tokio::task::yield_now() => {} + } +} diff --git a/tests/io_panic.rs b/tests/io_panic.rs new file mode 100644 index 0000000..922d7c0 --- /dev/null +++ b/tests/io_panic.rs @@ -0,0 +1,176 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery + +use std::task::{Context, Poll}; +use std::{error::Error, pin::Pin}; +use tokio::io::{self, split, AsyncRead, AsyncWrite, ReadBuf}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +struct RW; + +impl AsyncRead for RW { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + buf.put_slice(&[b'z']); + Poll::Ready(Ok(())) + } +} + +impl AsyncWrite for RW { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll<Result<usize, io::Error>> { + Poll::Ready(Ok(1)) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> { + Poll::Ready(Ok(())) + } +} + +#[cfg(unix)] +mod unix { + use std::os::unix::prelude::{AsRawFd, RawFd}; + + pub struct MockFd; + + impl AsRawFd for MockFd { + fn as_raw_fd(&self) -> RawFd { + 0 + } + } +} + +#[test] +fn read_buf_initialize_unfilled_to_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::<u8>::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + read_buf.initialize_unfilled_to(2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn read_buf_advance_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::<u8>::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + read_buf.advance(2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn read_buf_set_filled_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::<u8>::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + read_buf.set_filled(2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn read_buf_put_slice_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::<u8>::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + let new_slice = [0x40_u8, 0x41_u8]; + + read_buf.put_slice(&new_slice); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn unsplit_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let (r1, _w1) = split(RW); + let (_r2, w2) = split(RW); + r1.unsplit(w2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn async_fd_new_panic_caller() -> Result<(), Box<dyn Error>> { + use tokio::io::unix::AsyncFd; + use tokio::runtime::Builder; + + let panic_location_file = test_panic(|| { + // Runtime without `enable_io` so it has no IO driver set. + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(async { + let fd = unix::MockFd; + + let _ = AsyncFd::new(fd); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn async_fd_with_interest_panic_caller() -> Result<(), Box<dyn Error>> { + use tokio::io::unix::AsyncFd; + use tokio::io::Interest; + use tokio::runtime::Builder; + + let panic_location_file = test_panic(|| { + // Runtime without `enable_io` so it has no IO driver set. + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(async { + let fd = unix::MockFd; + + let _ = AsyncFd::with_interest(fd, Interest::READABLE); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/tests/io_read.rs b/tests/io_read.rs index cb1aa70..7f74c5e 100644 --- a/tests/io_read.rs +++ b/tests/io_read.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; @@ -8,6 +8,11 @@ use std::io; use std::pin::Pin; use std::task::{Context, Poll}; +mod support { + pub(crate) mod leaked_buffers; +} +use support::leaked_buffers::LeakedBuffers; + #[tokio::test] async fn read() { #[derive(Default)] @@ -37,16 +42,27 @@ async fn read() { assert_eq!(buf[..], b"hello world"[..]); } -struct BadAsyncRead; +struct BadAsyncRead { + leaked_buffers: LeakedBuffers, +} + +impl BadAsyncRead { + fn new() -> Self { + Self { + leaked_buffers: LeakedBuffers::new(), + } + } +} impl AsyncRead for BadAsyncRead { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll<io::Result<()>> { - *buf = ReadBuf::new(Box::leak(vec![0; buf.capacity()].into_boxed_slice())); + *buf = ReadBuf::new(unsafe { self.leaked_buffers.create(buf.capacity()) }); buf.advance(buf.capacity()); + Poll::Ready(Ok(())) } } @@ -55,5 +71,5 @@ impl AsyncRead for BadAsyncRead { #[should_panic] async fn read_buf_bad_async_read() { let mut buf = Vec::with_capacity(10); - BadAsyncRead.read_buf(&mut buf).await.unwrap(); + BadAsyncRead::new().read_buf(&mut buf).await.unwrap(); } diff --git a/tests/io_split.rs b/tests/io_split.rs index a012166..4cbd2a2 100644 --- a/tests/io_split.rs +++ b/tests/io_split.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery use tokio::io::{split, AsyncRead, AsyncWrite, ReadBuf, ReadHalf, WriteHalf}; diff --git a/tests/io_take.rs b/tests/io_take.rs index 683606f..fba01d2 100644 --- a/tests/io_take.rs +++ b/tests/io_take.rs @@ -1,9 +1,16 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery -use tokio::io::AsyncReadExt; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{self, AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; +mod support { + pub(crate) mod leaked_buffers; +} +use support::leaked_buffers::LeakedBuffers; + #[tokio::test] async fn take() { let mut buf = [0; 6]; @@ -14,3 +21,54 @@ async fn take() { assert_eq!(n, 4); assert_eq!(&buf, &b"hell\0\0"[..]); } + +#[tokio::test] +async fn issue_4435() { + let mut buf = [0; 8]; + let rd: &[u8] = b"hello world"; + + let rd = rd.take(4); + tokio::pin!(rd); + + let mut read_buf = ReadBuf::new(&mut buf); + read_buf.put_slice(b"AB"); + + futures::future::poll_fn(|cx| rd.as_mut().poll_read(cx, &mut read_buf)) + .await + .unwrap(); + assert_eq!(&buf, &b"ABhell\0\0"[..]); +} + +struct BadReader { + leaked_buffers: LeakedBuffers, +} + +impl BadReader { + fn new() -> Self { + Self { + leaked_buffers: LeakedBuffers::new(), + } + } +} + +impl AsyncRead for BadReader { + fn poll_read( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + read_buf: &mut ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + let mut buf = ReadBuf::new(unsafe { self.leaked_buffers.create(10) }); + buf.put_slice(&[123; 10]); + *read_buf = buf; + + Poll::Ready(Ok(())) + } +} + +#[tokio::test] +#[should_panic] +async fn bad_reader_fails() { + let mut buf = Vec::with_capacity(10); + + BadReader::new().take(10).read_buf(&mut buf).await.unwrap(); +} diff --git a/tests/io_util_empty.rs b/tests/io_util_empty.rs new file mode 100644 index 0000000..e49cd17 --- /dev/null +++ b/tests/io_util_empty.rs @@ -0,0 +1,32 @@ +#![cfg(feature = "full")] +use tokio::io::{AsyncBufReadExt, AsyncReadExt}; + +#[tokio::test] +async fn empty_read_is_cooperative() { + tokio::select! { + biased; + + _ = async { + loop { + let mut buf = [0u8; 4096]; + let _ = tokio::io::empty().read(&mut buf).await; + } + } => {}, + _ = tokio::task::yield_now() => {} + } +} + +#[tokio::test] +async fn empty_buf_reads_are_cooperative() { + tokio::select! { + biased; + + _ = async { + loop { + let mut buf = String::new(); + let _ = tokio::io::empty().read_line(&mut buf).await; + } + } => {}, + _ = tokio::task::yield_now() => {} + } +} diff --git a/tests/join_handle_panic.rs b/tests/join_handle_panic.rs new file mode 100644 index 0000000..bc34fd0 --- /dev/null +++ b/tests/join_handle_panic.rs @@ -0,0 +1,20 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery + +struct PanicsOnDrop; + +impl Drop for PanicsOnDrop { + fn drop(&mut self) { + panic!("I told you so"); + } +} + +#[tokio::test] +async fn test_panics_do_not_propagate_when_dropping_join_handle() { + let join_handle = tokio::spawn(async move { PanicsOnDrop }); + + // only drop the JoinHandle when the task has completed + // (which is difficult to synchronize precisely) + tokio::time::sleep(std::time::Duration::from_millis(3)).await; + drop(join_handle); +} diff --git a/tests/macros_join.rs b/tests/macros_join.rs index 169e898..9e4d234 100644 --- a/tests/macros_join.rs +++ b/tests/macros_join.rs @@ -1,36 +1,48 @@ -#![allow(clippy::blacklisted_name)] -use tokio::sync::oneshot; +#![cfg(feature = "macros")] +#![allow(clippy::disallowed_names)] +use std::sync::Arc; + +#[cfg(tokio_wasm_not_wasi)] +#[cfg(target_pointer_width = "64")] +use wasm_bindgen_test::wasm_bindgen_test as test; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; + +use tokio::sync::{oneshot, Semaphore}; use tokio_test::{assert_pending, assert_ready, task}; -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_lit_expr_comma() { let foo = tokio::join!(async { 1 },); assert_eq!(foo, (1,)); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_lit_expr_no_comma() { let foo = tokio::join!(async { 1 }); assert_eq!(foo, (1,)); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_two_lit_expr_comma() { let foo = tokio::join!(async { 1 }, async { 2 },); assert_eq!(foo, (1, 2)); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_two_lit_expr_no_comma() { let foo = tokio::join!(async { 1 }, async { 2 }); assert_eq!(foo, (1, 2)); } -#[tokio::test] +#[maybe_tokio_test] async fn two_await() { let (tx1, rx1) = oneshot::channel::<&str>(); let (tx2, rx2) = oneshot::channel::<u32>(); @@ -53,6 +65,8 @@ async fn two_await() { } #[test] +#[cfg(target_pointer_width = "64")] +#[ignore = "Android: ignore until the compiler is updated. aliceryhl@ says these tests assume latest stable compiler."] fn join_size() { use futures::future; use std::mem; @@ -61,12 +75,82 @@ fn join_size() { let ready = future::ready(0i32); tokio::join!(ready) }; - assert_eq!(mem::size_of_val(&fut), 16); + assert_eq!(mem::size_of_val(&fut), 32); let fut = async { let ready1 = future::ready(0i32); let ready2 = future::ready(0i32); tokio::join!(ready1, ready2) }; - assert_eq!(mem::size_of_val(&fut), 28); + assert_eq!(mem::size_of_val(&fut), 40); +} + +async fn non_cooperative_task(permits: Arc<Semaphore>) -> usize { + let mut exceeded_budget = 0; + + for _ in 0..5 { + // Another task should run after this task uses its whole budget + for _ in 0..128 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + } + + exceeded_budget += 1; + } + + exceeded_budget +} + +async fn poor_little_task(permits: Arc<Semaphore>) -> usize { + let mut how_many_times_i_got_to_run = 0; + + for _ in 0..5 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + how_many_times_i_got_to_run += 1; + } + + how_many_times_i_got_to_run +} + +#[tokio::test] +async fn join_does_not_allow_tasks_to_starve() { + let permits = Arc::new(Semaphore::new(1)); + + // non_cooperative_task should yield after its budget is exceeded and then poor_little_task should run. + let (non_cooperative_result, little_task_result) = tokio::join!( + non_cooperative_task(Arc::clone(&permits)), + poor_little_task(permits) + ); + + assert_eq!(5, non_cooperative_result); + assert_eq!(5, little_task_result); +} + +#[tokio::test] +async fn a_different_future_is_polled_first_every_time_poll_fn_is_polled() { + let poll_order = Arc::new(std::sync::Mutex::new(vec![])); + + let fut = |x, poll_order: Arc<std::sync::Mutex<Vec<i32>>>| async move { + for _ in 0..4 { + { + let mut guard = poll_order.lock().unwrap(); + + guard.push(x); + } + + tokio::task::yield_now().await; + } + }; + + tokio::join!( + fut(1, Arc::clone(&poll_order)), + fut(2, Arc::clone(&poll_order)), + fut(3, Arc::clone(&poll_order)), + ); + + // Each time the future created by join! is polled, it should start + // by polling a different future first. + assert_eq!( + vec![1, 2, 3, 2, 3, 1, 3, 1, 2, 1, 2, 3], + *poll_order.lock().unwrap() + ); } diff --git a/tests/macros_pin.rs b/tests/macros_pin.rs index da6e0be..e99a3ee 100644 --- a/tests/macros_pin.rs +++ b/tests/macros_pin.rs @@ -1,7 +1,15 @@ +#![cfg(feature = "macros")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; + async fn one() {} async fn two() {} -#[tokio::test] +#[maybe_tokio_test] async fn multi_pin() { tokio::pin! { let f1 = one(); diff --git a/tests/macros_rename_test.rs b/tests/macros_rename_test.rs new file mode 100644 index 0000000..90a86f9 --- /dev/null +++ b/tests/macros_rename_test.rs @@ -0,0 +1,26 @@ +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threading + +#[allow(unused_imports)] +use std as tokio; + +use ::tokio as tokio1; + +async fn compute() -> usize { + let join = tokio1::spawn(async { 1 }); + join.await.unwrap() +} + +#[tokio1::main(crate = "tokio1")] +async fn compute_main() -> usize { + compute().await +} + +#[test] +fn crate_rename_main() { + assert_eq!(1, compute_main()); +} + +#[tokio1::test(crate = "tokio1")] +async fn crate_rename_test() { + assert_eq!(1, compute().await); +} diff --git a/tests/macros_select.rs b/tests/macros_select.rs index b4f8544..26d6fec 100644 --- a/tests/macros_select.rs +++ b/tests/macros_select.rs @@ -1,12 +1,19 @@ -#![allow(clippy::blacklisted_name)] -use tokio::sync::{mpsc, oneshot}; -use tokio::task; +#![cfg(feature = "macros")] +#![allow(clippy::disallowed_names)] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; + +use tokio::sync::oneshot; use tokio_test::{assert_ok, assert_pending, assert_ready}; use futures::future::poll_fn; use std::task::Poll::Ready; -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_lit_expr_comma() { let foo = tokio::select! { foo = async { 1 } => foo, @@ -15,7 +22,7 @@ async fn sync_one_lit_expr_comma() { assert_eq!(foo, 1); } -#[tokio::test] +#[maybe_tokio_test] async fn nested_one() { let foo = tokio::select! { foo = async { 1 } => tokio::select! { @@ -26,7 +33,7 @@ async fn nested_one() { assert_eq!(foo, 1); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_lit_expr_no_comma() { let foo = tokio::select! { foo = async { 1 } => foo @@ -35,7 +42,7 @@ async fn sync_one_lit_expr_no_comma() { assert_eq!(foo, 1); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_lit_expr_block() { let foo = tokio::select! { foo = async { 1 } => { foo } @@ -44,7 +51,7 @@ async fn sync_one_lit_expr_block() { assert_eq!(foo, 1); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_await() { let foo = tokio::select! { foo = one() => foo, @@ -53,7 +60,7 @@ async fn sync_one_await() { assert_eq!(foo, 1); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_ident() { let one = one(); @@ -64,7 +71,7 @@ async fn sync_one_ident() { assert_eq!(foo, 1); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_two() { use std::cell::Cell; @@ -85,7 +92,7 @@ async fn sync_two() { assert!(res == 1 || res == 2); } -#[tokio::test] +#[maybe_tokio_test] async fn drop_in_fut() { let s = "hello".to_string(); @@ -100,7 +107,8 @@ async fn drop_in_fut() { assert_eq!(res, 1); } -#[tokio::test] +#[maybe_tokio_test] +#[cfg(feature = "full")] async fn one_ready() { let (tx1, rx1) = oneshot::channel::<i32>(); let (_tx2, rx2) = oneshot::channel::<i32>(); @@ -117,20 +125,23 @@ async fn one_ready() { assert_eq!(1, v); } -#[tokio::test] +#[maybe_tokio_test] +#[cfg(feature = "full")] async fn select_streams() { + use tokio::sync::mpsc; + let (tx1, mut rx1) = mpsc::unbounded_channel::<i32>(); let (tx2, mut rx2) = mpsc::unbounded_channel::<i32>(); tokio::spawn(async move { assert_ok!(tx2.send(1)); - task::yield_now().await; + tokio::task::yield_now().await; assert_ok!(tx1.send(2)); - task::yield_now().await; + tokio::task::yield_now().await; assert_ok!(tx2.send(3)); - task::yield_now().await; + tokio::task::yield_now().await; drop((tx1, tx2)); }); @@ -156,7 +167,7 @@ async fn select_streams() { assert_eq!(&msgs[..], &[1, 2, 3]); } -#[tokio::test] +#[maybe_tokio_test] async fn move_uncompleted_futures() { let (tx1, mut rx1) = oneshot::channel::<i32>(); let (tx2, mut rx2) = oneshot::channel::<i32>(); @@ -182,7 +193,7 @@ async fn move_uncompleted_futures() { assert!(ran); } -#[tokio::test] +#[maybe_tokio_test] async fn nested() { let res = tokio::select! { x = async { 1 } => { @@ -195,7 +206,8 @@ async fn nested() { assert_eq!(res, 3); } -#[tokio::test] +#[maybe_tokio_test] +#[cfg(target_pointer_width = "64")] async fn struct_size() { use futures::future; use std::mem; @@ -208,7 +220,7 @@ async fn struct_size() { } }; - assert!(mem::size_of_val(&fut) <= 32); + assert_eq!(mem::size_of_val(&fut), 40); let fut = async { let ready1 = future::ready(0i32); @@ -220,7 +232,7 @@ async fn struct_size() { } }; - assert!(mem::size_of_val(&fut) <= 40); + assert_eq!(mem::size_of_val(&fut), 48); let fut = async { let ready1 = future::ready(0i32); @@ -234,10 +246,10 @@ async fn struct_size() { } }; - assert!(mem::size_of_val(&fut) <= 48); + assert_eq!(mem::size_of_val(&fut), 56); } -#[tokio::test] +#[maybe_tokio_test] async fn mutable_borrowing_future_with_same_borrow_in_block() { let mut value = 234; @@ -251,7 +263,7 @@ async fn mutable_borrowing_future_with_same_borrow_in_block() { assert!(value >= 234); } -#[tokio::test] +#[maybe_tokio_test] async fn mutable_borrowing_future_with_same_borrow_in_block_and_else() { let mut value = 234; @@ -268,7 +280,7 @@ async fn mutable_borrowing_future_with_same_borrow_in_block_and_else() { assert!(value >= 234); } -#[tokio::test] +#[maybe_tokio_test] async fn future_panics_after_poll() { use tokio_test::task; @@ -298,7 +310,7 @@ async fn future_panics_after_poll() { assert_eq!(1, res); } -#[tokio::test] +#[maybe_tokio_test] async fn disable_with_if() { use tokio_test::task; @@ -320,7 +332,7 @@ async fn disable_with_if() { assert_ready!(f.poll()); } -#[tokio::test] +#[maybe_tokio_test] async fn join_with_select() { use tokio_test::task; @@ -356,6 +368,7 @@ async fn join_with_select() { } #[tokio::test] +#[cfg(feature = "full")] async fn use_future_in_if_condition() { use tokio::time::{self, Duration}; @@ -369,6 +382,7 @@ async fn use_future_in_if_condition() { } #[tokio::test] +#[cfg(feature = "full")] async fn use_future_in_if_condition_biased() { use tokio::time::{self, Duration}; @@ -382,7 +396,7 @@ async fn use_future_in_if_condition_biased() { } } -#[tokio::test] +#[maybe_tokio_test] async fn many_branches() { let num = tokio::select! { x = async { 1 } => x, @@ -448,12 +462,13 @@ async fn many_branches() { x = async { 1 } => x, x = async { 1 } => x, x = async { 1 } => x, + x = async { 1 } => x, }; assert_eq!(1, num); } -#[tokio::test] +#[maybe_tokio_test] async fn never_branch_no_warnings() { let t = tokio::select! { _ = async_never() => 0, @@ -474,7 +489,7 @@ async fn async_never() -> ! { } // From https://github.com/tokio-rs/tokio/issues/2857 -#[tokio::test] +#[maybe_tokio_test] async fn mut_on_left_hand_side() { let v = async move { let ok = async { 1 }; @@ -490,7 +505,7 @@ async fn mut_on_left_hand_side() { assert_eq!(v, 2); } -#[tokio::test] +#[maybe_tokio_test] async fn biased_one_not_ready() { let (_tx1, rx1) = oneshot::channel::<i32>(); let (tx2, rx2) = oneshot::channel::<i32>(); @@ -514,7 +529,8 @@ async fn biased_one_not_ready() { assert_eq!(2, v); } -#[tokio::test] +#[maybe_tokio_test] +#[cfg(feature = "full")] async fn biased_eventually_ready() { use tokio::task::yield_now; @@ -560,7 +576,7 @@ pub async fn default_numeric_fallback() { } // https://github.com/tokio-rs/tokio/issues/4182 -#[tokio::test] +#[maybe_tokio_test] async fn mut_ref_patterns() { tokio::select! { Some(mut foo) = async { Some("1".to_string()) } => { @@ -584,3 +600,66 @@ async fn mut_ref_patterns() { }, }; } + +#[cfg(tokio_unstable)] +mod unstable { + use tokio::runtime::RngSeed; + + #[test] + fn deterministic_select_current_thread() { + let seed = b"bytes used to generate seed"; + let rt1 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values = rt1.block_on(async { (select_0_to_9().await, select_0_to_9().await) }); + + let rt2 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values = rt2.block_on(async { (select_0_to_9().await, select_0_to_9().await) }); + + assert_eq!(rt1_values, rt2_values); + } + + #[test] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + fn deterministic_select_multi_thread() { + let seed = b"bytes used to generate seed"; + let rt1 = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values = rt1.block_on(async { + let _ = tokio::spawn(async { (select_0_to_9().await, select_0_to_9().await) }).await; + }); + + let rt2 = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values = rt2.block_on(async { + let _ = tokio::spawn(async { (select_0_to_9().await, select_0_to_9().await) }).await; + }); + + assert_eq!(rt1_values, rt2_values); + } + + async fn select_0_to_9() -> u32 { + tokio::select!( + x = async { 0 } => x, + x = async { 1 } => x, + x = async { 2 } => x, + x = async { 3 } => x, + x = async { 4 } => x, + x = async { 5 } => x, + x = async { 6 } => x, + x = async { 7 } => x, + x = async { 8 } => x, + x = async { 9 } => x, + ) + } +} diff --git a/tests/macros_test.rs b/tests/macros_test.rs index bca2c91..85279b7 100644 --- a/tests/macros_test.rs +++ b/tests/macros_test.rs @@ -1,3 +1,5 @@ +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threading + use tokio::test; #[test] @@ -46,3 +48,41 @@ pub async fn issue_4175_test() -> std::io::Result<()> { return Ok(()); panic!(); } + +// https://github.com/tokio-rs/tokio/issues/4175 +#[allow(clippy::let_unit_value)] +pub mod clippy_semicolon_if_nothing_returned { + #![deny(clippy::semicolon_if_nothing_returned)] + + #[tokio::main] + pub async fn local() { + let _x = (); + } + #[tokio::main] + pub async fn item() { + fn _f() {} + } + #[tokio::main] + pub async fn semi() { + panic!(); + } + #[tokio::main] + pub async fn empty() { + // To trigger clippy::semicolon_if_nothing_returned lint, the block needs to contain newline. + } +} + +// https://github.com/tokio-rs/tokio/issues/5243 +pub mod issue_5243 { + macro_rules! mac { + (async fn $name:ident() $b:block) => { + #[::tokio::test] + async fn $name() { + $b + } + }; + } + mac!( + async fn foo() {} + ); +} diff --git a/tests/macros_try_join.rs b/tests/macros_try_join.rs index a925153..6c43222 100644 --- a/tests/macros_try_join.rs +++ b/tests/macros_try_join.rs @@ -1,36 +1,46 @@ -#![allow(clippy::blacklisted_name)] -use tokio::sync::oneshot; +#![cfg(feature = "macros")] +#![allow(clippy::disallowed_names)] + +use std::sync::Arc; + +use tokio::sync::{oneshot, Semaphore}; use tokio_test::{assert_pending, assert_ready, task}; -#[tokio::test] +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; + +#[maybe_tokio_test] async fn sync_one_lit_expr_comma() { let foo = tokio::try_join!(async { ok(1) },); assert_eq!(foo, Ok((1,))); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_one_lit_expr_no_comma() { let foo = tokio::try_join!(async { ok(1) }); assert_eq!(foo, Ok((1,))); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_two_lit_expr_comma() { let foo = tokio::try_join!(async { ok(1) }, async { ok(2) },); assert_eq!(foo, Ok((1, 2))); } -#[tokio::test] +#[maybe_tokio_test] async fn sync_two_lit_expr_no_comma() { let foo = tokio::try_join!(async { ok(1) }, async { ok(2) }); assert_eq!(foo, Ok((1, 2))); } -#[tokio::test] +#[maybe_tokio_test] async fn two_await() { let (tx1, rx1) = oneshot::channel::<&str>(); let (tx2, rx2) = oneshot::channel::<u32>(); @@ -51,7 +61,7 @@ async fn two_await() { assert_eq!(Ok(("hello", 123)), res); } -#[tokio::test] +#[maybe_tokio_test] async fn err_abort_early() { let (tx1, rx1) = oneshot::channel::<&str>(); let (tx2, rx2) = oneshot::channel::<u32>(); @@ -78,6 +88,7 @@ async fn err_abort_early() { } #[test] +#[cfg(target_pointer_width = "64")] fn join_size() { use futures::future; use std::mem; @@ -86,16 +97,89 @@ fn join_size() { let ready = future::ready(ok(0i32)); tokio::try_join!(ready) }; - assert_eq!(mem::size_of_val(&fut), 16); + assert_eq!(mem::size_of_val(&fut), 32); let fut = async { let ready1 = future::ready(ok(0i32)); let ready2 = future::ready(ok(0i32)); tokio::try_join!(ready1, ready2) }; - assert_eq!(mem::size_of_val(&fut), 28); + assert_eq!(mem::size_of_val(&fut), 48); } fn ok<T>(val: T) -> Result<T, ()> { Ok(val) } + +async fn non_cooperative_task(permits: Arc<Semaphore>) -> Result<usize, String> { + let mut exceeded_budget = 0; + + for _ in 0..5 { + // Another task should run after this task uses its whole budget + for _ in 0..128 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + } + + exceeded_budget += 1; + } + + Ok(exceeded_budget) +} + +async fn poor_little_task(permits: Arc<Semaphore>) -> Result<usize, String> { + let mut how_many_times_i_got_to_run = 0; + + for _ in 0..5 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + + how_many_times_i_got_to_run += 1; + } + + Ok(how_many_times_i_got_to_run) +} + +#[tokio::test] +async fn try_join_does_not_allow_tasks_to_starve() { + let permits = Arc::new(Semaphore::new(10)); + + // non_cooperative_task should yield after its budget is exceeded and then poor_little_task should run. + let result = tokio::try_join!( + non_cooperative_task(Arc::clone(&permits)), + poor_little_task(permits) + ); + + let (non_cooperative_result, little_task_result) = result.unwrap(); + + assert_eq!(5, non_cooperative_result); + assert_eq!(5, little_task_result); +} + +#[tokio::test] +async fn a_different_future_is_polled_first_every_time_poll_fn_is_polled() { + let poll_order = Arc::new(std::sync::Mutex::new(vec![])); + + let fut = |x, poll_order: Arc<std::sync::Mutex<Vec<i32>>>| async move { + for _ in 0..4 { + { + let mut guard = poll_order.lock().unwrap(); + + guard.push(x); + } + + tokio::task::yield_now().await; + } + }; + + tokio::join!( + fut(1, Arc::clone(&poll_order)), + fut(2, Arc::clone(&poll_order)), + fut(3, Arc::clone(&poll_order)), + ); + + // Each time the future created by join! is polled, it should start + // by polling a different future first. + assert_eq!( + vec![1, 2, 3, 2, 3, 1, 3, 1, 2, 1, 2, 3], + *poll_order.lock().unwrap() + ); +} diff --git a/tests/net_bind_resource.rs b/tests/net_bind_resource.rs index d4a0b8d..1c604aa 100644 --- a/tests/net_bind_resource.rs +++ b/tests/net_bind_resource.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery or bind use tokio::net::TcpListener; diff --git a/tests/net_lookup_host.rs b/tests/net_lookup_host.rs index 4d06402..2b346e6 100644 --- a/tests/net_lookup_host.rs +++ b/tests/net_lookup_host.rs @@ -1,3 +1,5 @@ +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support direct socket operations + use tokio::net; use tokio_test::assert_ok; diff --git a/tests/named_pipe.rs b/tests/net_named_pipe.rs index 2055c3c..c421224 100644 --- a/tests/named_pipe.rs +++ b/tests/net_named_pipe.rs @@ -8,7 +8,7 @@ use std::time::Duration; use tokio::io::AsyncWriteExt; use tokio::net::windows::named_pipe::{ClientOptions, PipeMode, ServerOptions}; use tokio::time; -use winapi::shared::winerror; +use windows_sys::Win32::Foundation::{ERROR_NO_DATA, ERROR_PIPE_BUSY, NO_ERROR, UNICODE_STRING}; #[tokio::test] async fn test_named_pipe_client_drop() -> io::Result<()> { @@ -25,7 +25,7 @@ async fn test_named_pipe_client_drop() -> io::Result<()> { // instance will be broken because client is gone match server.write_all(b"ping").await { - Err(e) if e.raw_os_error() == Some(winerror::ERROR_NO_DATA as i32) => (), + Err(e) if e.raw_os_error() == Some(ERROR_NO_DATA as i32) => (), x => panic!("{:?}", x), } @@ -120,7 +120,7 @@ async fn test_named_pipe_multi_client() -> io::Result<()> { let client = loop { match ClientOptions::new().open(PIPE_NAME) { Ok(client) => break client, - Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), + Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), Err(e) if e.kind() == io::ErrorKind::NotFound => (), Err(e) => return Err(e), } @@ -249,7 +249,7 @@ async fn test_named_pipe_multi_client_ready() -> io::Result<()> { let client = loop { match ClientOptions::new().open(PIPE_NAME) { Ok(client) => break client, - Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), + Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), Err(e) if e.kind() == io::ErrorKind::NotFound => (), Err(e) => return Err(e), } @@ -341,12 +341,38 @@ async fn test_named_pipe_mode_message() -> io::Result<()> { Ok(()) } +// This tests `NamedPipeServer::connect` with various access settings. +#[tokio::test] +async fn test_named_pipe_access() -> io::Result<()> { + const PIPE_NAME: &str = r"\\.\pipe\test-named-pipe-access"; + + for (inb, outb) in [(true, true), (true, false), (false, true)] { + let (tx, rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let s = ServerOptions::new() + .access_inbound(inb) + .access_outbound(outb) + .create(PIPE_NAME)?; + let mut connect_fut = tokio_test::task::spawn(s.connect()); + assert!(connect_fut.poll().is_pending()); + tx.send(()).unwrap(); + connect_fut.await + }); + + // Wait for the server to call connect. + rx.await.unwrap(); + let _ = ClientOptions::new().read(outb).write(inb).open(PIPE_NAME)?; + + server.await??; + } + Ok(()) +} + fn num_instances(pipe_name: impl AsRef<str>) -> io::Result<u32> { use ntapi::ntioapi; - use winapi::shared::ntdef; let mut name = pipe_name.as_ref().encode_utf16().collect::<Vec<_>>(); - let mut name = ntdef::UNICODE_STRING { + let mut name = UNICODE_STRING { Length: (name.len() * mem::size_of::<u16>()) as u16, MaximumLength: (name.len() * mem::size_of::<u16>()) as u16, Buffer: name.as_mut_ptr(), @@ -366,12 +392,12 @@ fn num_instances(pipe_name: impl AsRef<str>) -> io::Result<u32> { 1024, ntioapi::FileDirectoryInformation, 0, - &mut name, + &mut name as *mut _ as _, 0, ) }; - if status as u32 != winerror::NO_ERROR { + if status as u32 != NO_ERROR { return Err(io::Error::last_os_error()); } diff --git a/tests/net_panic.rs b/tests/net_panic.rs new file mode 100644 index 0000000..fc2209a --- /dev/null +++ b/tests/net_panic.rs @@ -0,0 +1,188 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] + +use std::error::Error; +use tokio::net::{TcpListener, TcpStream}; +use tokio::runtime::{Builder, Runtime}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn udp_socket_from_std_panic_caller() -> Result<(), Box<dyn Error>> { + use std::net::SocketAddr; + use tokio::net::UdpSocket; + + let addr = "127.0.0.1:0".parse::<SocketAddr>().unwrap(); + let std_sock = std::net::UdpSocket::bind(addr).unwrap(); + std_sock.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _sock = UdpSocket::from_std(std_sock); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn tcp_listener_from_std_panic_caller() -> Result<(), Box<dyn Error>> { + let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + std_listener.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = TcpListener::from_std(std_listener); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn tcp_stream_from_std_panic_caller() -> Result<(), Box<dyn Error>> { + let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + + let std_stream = std::net::TcpStream::connect(std_listener.local_addr().unwrap()).unwrap(); + std_stream.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = TcpStream::from_std(std_stream); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_listener_bind_panic_caller() -> Result<(), Box<dyn Error>> { + use tokio::net::UnixListener; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixListener::bind(&sock_path); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_listener_from_std_panic_caller() -> Result<(), Box<dyn Error>> { + use tokio::net::UnixListener; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + let std_listener = std::os::unix::net::UnixListener::bind(&sock_path).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixListener::from_std(std_listener); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_stream_from_std_panic_caller() -> Result<(), Box<dyn Error>> { + use tokio::net::UnixStream; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + let _std_listener = std::os::unix::net::UnixListener::bind(&sock_path).unwrap(); + let std_stream = std::os::unix::net::UnixStream::connect(&sock_path).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixStream::from_std(std_stream); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_datagram_from_std_panic_caller() -> Result<(), Box<dyn Error>> { + use std::os::unix::net::UnixDatagram as StdUDS; + use tokio::net::UnixDatagram; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + + // Bind the socket to a filesystem path + // /let socket_path = tmp.path().join("socket"); + let std_socket = StdUDS::bind(&sock_path).unwrap(); + std_socket.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(move || { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixDatagram::from_std(std_socket); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(windows)] +fn server_options_max_instances_panic_caller() -> Result<(), Box<dyn Error>> { + use tokio::net::windows::named_pipe::ServerOptions; + + let panic_location_file = test_panic(move || { + let rt = runtime_without_io(); + rt.block_on(async { + let mut options = ServerOptions::new(); + options.max_instances(255); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +// Runtime without `enable_io` so it has no IO driver set. +fn runtime_without_io() -> Runtime { + Builder::new_current_thread().build().unwrap() +} diff --git a/tests/no_rt.rs b/tests/no_rt.rs index 6845850..747fab6 100644 --- a/tests/no_rt.rs +++ b/tests/no_rt.rs @@ -1,3 +1,5 @@ +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery + use tokio::net::TcpStream; use tokio::sync::oneshot; use tokio::time::{timeout, Duration}; diff --git a/tests/process_raw_handle.rs b/tests/process_raw_handle.rs index 727e66d..fbe22b0 100644 --- a/tests/process_raw_handle.rs +++ b/tests/process_raw_handle.rs @@ -3,7 +3,7 @@ #![cfg(windows)] use tokio::process::Command; -use winapi::um::processthreadsapi::GetProcessId; +use windows_sys::Win32::System::Threading::GetProcessId; #[tokio::test] async fn obtain_raw_handle() { diff --git a/tests/process_smoke.rs b/tests/process_smoke.rs index fae5793..81d2020 100644 --- a/tests/process_smoke.rs +++ b/tests/process_smoke.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi cannot run system commands use tokio::process::Command; use tokio_test::assert_ok; diff --git a/tests/rt_basic.rs b/tests/rt_basic.rs index 70056b1..6caf0a4 100644 --- a/tests/rt_basic.rs +++ b/tests/rt_basic.rs @@ -16,6 +16,15 @@ mod support { pub(crate) mod mpsc_stream; } +macro_rules! cfg_metrics { + ($($t:tt)*) => { + #[cfg(tokio_unstable)] + { + $( $t )* + } + } +} + #[test] fn spawned_task_does_not_progress_without_block_on() { let (tx, mut rx) = oneshot::channel(); @@ -169,6 +178,105 @@ fn drop_tasks_in_context() { } #[test] +#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] +#[should_panic(expected = "boom")] +fn wake_in_drop_after_panic() { + let (tx, rx) = oneshot::channel::<()>(); + + struct WakeOnDrop(Option<oneshot::Sender<()>>); + + impl Drop for WakeOnDrop { + fn drop(&mut self) { + self.0.take().unwrap().send(()).unwrap(); + } + } + + let rt = rt(); + + rt.spawn(async move { + let _wake_on_drop = WakeOnDrop(Some(tx)); + // wait forever + futures::future::pending::<()>().await; + }); + + let _join = rt.spawn(async move { rx.await }); + + rt.block_on(async { + tokio::task::yield_now().await; + panic!("boom"); + }); +} + +#[test] +fn spawn_two() { + let rt = rt(); + + let out = rt.block_on(async { + let (tx, rx) = oneshot::channel(); + + tokio::spawn(async move { + tokio::spawn(async move { + tx.send("ZOMG").unwrap(); + }); + }); + + assert_ok!(rx.await) + }); + + assert_eq!(out, "ZOMG"); + + cfg_metrics! { + let metrics = rt.metrics(); + drop(rt); + assert_eq!(0, metrics.remote_schedule_count()); + + let mut local = 0; + for i in 0..metrics.num_workers() { + local += metrics.worker_local_schedule_count(i); + } + + assert_eq!(2, local); + } +} + +#[cfg_attr(tokio_wasi, ignore = "WASI: std::thread::spawn not supported")] +#[test] +fn spawn_remote() { + let rt = rt(); + + let out = rt.block_on(async { + let (tx, rx) = oneshot::channel(); + + let handle = tokio::spawn(async move { + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(10)); + tx.send("ZOMG").unwrap(); + }); + + rx.await.unwrap() + }); + + handle.await.unwrap() + }); + + assert_eq!(out, "ZOMG"); + + cfg_metrics! { + let metrics = rt.metrics(); + drop(rt); + assert_eq!(1, metrics.remote_schedule_count()); + + let mut local = 0; + for i in 0..metrics.num_workers() { + local += metrics.worker_local_schedule_count(i); + } + + assert_eq!(1, local); + } +} + +#[test] +#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] #[should_panic( expected = "A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers." )] @@ -183,6 +291,155 @@ fn timeout_panics_when_no_time_handle() { }); } +#[cfg(tokio_unstable)] +mod unstable { + use tokio::runtime::{Builder, RngSeed, UnhandledPanic}; + + #[test] + #[should_panic( + expected = "a spawned task panicked and the runtime is configured to shut down on unhandled panic" + )] + fn shutdown_on_panic() { + let rt = Builder::new_current_thread() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .build() + .unwrap(); + + rt.block_on(async { + tokio::spawn(async { + panic!("boom"); + }); + + futures::future::pending::<()>().await; + }) + } + + #[test] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] + fn spawns_do_nothing() { + use std::sync::Arc; + + let rt = Builder::new_current_thread() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .build() + .unwrap(); + + let rt1 = Arc::new(rt); + let rt2 = rt1.clone(); + + let _ = std::thread::spawn(move || { + rt2.block_on(async { + tokio::spawn(async { + panic!("boom"); + }); + + futures::future::pending::<()>().await; + }) + }) + .join(); + + let task = rt1.spawn(async {}); + let res = futures::executor::block_on(task); + assert!(res.is_err()); + } + + #[test] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] + fn shutdown_all_concurrent_block_on() { + const N: usize = 2; + use std::sync::{mpsc, Arc}; + + let rt = Builder::new_current_thread() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .build() + .unwrap(); + + let rt = Arc::new(rt); + let mut ths = vec![]; + let (tx, rx) = mpsc::channel(); + + for _ in 0..N { + let rt = rt.clone(); + let tx = tx.clone(); + ths.push(std::thread::spawn(move || { + rt.block_on(async { + tx.send(()).unwrap(); + futures::future::pending::<()>().await; + }); + })); + } + + for _ in 0..N { + rx.recv().unwrap(); + } + + rt.spawn(async { + panic!("boom"); + }); + + for th in ths { + assert!(th.join().is_err()); + } + } + + #[test] + fn rng_seed() { + let seed = b"bytes used to generate seed"; + let rt1 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values = rt1.block_on(async { + let rand_1 = tokio::macros::support::thread_rng_n(100); + let rand_2 = tokio::macros::support::thread_rng_n(100); + + (rand_1, rand_2) + }); + + let rt2 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values = rt2.block_on(async { + let rand_1 = tokio::macros::support::thread_rng_n(100); + let rand_2 = tokio::macros::support::thread_rng_n(100); + + (rand_1, rand_2) + }); + + assert_eq!(rt1_values, rt2_values); + } + + #[test] + fn rng_seed_multi_enter() { + let seed = b"bytes used to generate seed"; + + fn two_rand_values() -> (u32, u32) { + let rand_1 = tokio::macros::support::thread_rng_n(100); + let rand_2 = tokio::macros::support::thread_rng_n(100); + + (rand_1, rand_2) + } + + let rt1 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values_1 = rt1.block_on(async { two_rand_values() }); + let rt1_values_2 = rt1.block_on(async { two_rand_values() }); + + let rt2 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values_1 = rt2.block_on(async { two_rand_values() }); + let rt2_values_2 = rt2.block_on(async { two_rand_values() }); + + assert_eq!(rt1_values_1, rt2_values_1); + assert_eq!(rt1_values_2, rt2_values_2); + } +} + fn rt() -> Runtime { tokio::runtime::Builder::new_current_thread() .enable_all() diff --git a/tests/rt_common.rs b/tests/rt_common.rs index e5fc7a9..3892998 100644 --- a/tests/rt_common.rs +++ b/tests/rt_common.rs @@ -2,13 +2,16 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "full")] -// Tests to run on both current-thread & thread-pool runtime variants. +// Tests to run on both current-thread & multi-thread runtime variants. macro_rules! rt_test { ($($t:tt)*) => { mod current_thread_scheduler { $($t)* + #[cfg(not(target_os="wasi"))] + const NUM_WORKERS: usize = 1; + fn rt() -> Arc<Runtime> { tokio::runtime::Builder::new_current_thread() .enable_all() @@ -18,9 +21,12 @@ macro_rules! rt_test { } } + #[cfg(not(tokio_wasi))] // Wasi doesn't support threads mod threaded_scheduler_4_threads { $($t)* + const NUM_WORKERS: usize = 4; + fn rt() -> Arc<Runtime> { tokio::runtime::Builder::new_multi_thread() .worker_threads(4) @@ -31,9 +37,12 @@ macro_rules! rt_test { } } + #[cfg(not(tokio_wasi))] // Wasi doesn't support threads mod threaded_scheduler_1_thread { $($t)* + const NUM_WORKERS: usize = 1; + fn rt() -> Arc<Runtime> { tokio::runtime::Builder::new_multi_thread() .worker_threads(1) @@ -55,18 +64,30 @@ fn send_sync_bound() { } rt_test! { - use tokio::net::{TcpListener, TcpStream, UdpSocket}; + #[cfg(not(target_os="wasi"))] + use tokio::net::{TcpListener, TcpStream}; + #[cfg(not(target_os="wasi"))] use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::runtime::Runtime; use tokio::sync::oneshot; use tokio::{task, time}; - use tokio_test::{assert_err, assert_ok}; + + #[cfg(not(target_os="wasi"))] + use tokio_test::assert_err; + use tokio_test::assert_ok; use futures::future::poll_fn; use std::future::Future; use std::pin::Pin; - use std::sync::{mpsc, Arc}; + + #[cfg(not(target_os="wasi"))] + use std::sync::mpsc; + + use std::sync::Arc; use std::task::{Context, Poll}; + + #[cfg(not(target_os="wasi"))] use std::thread; use std::time::{Duration, Instant}; @@ -83,6 +104,7 @@ rt_test! { } + #[cfg(not(target_os="wasi"))] #[test] fn block_on_async() { let rt = rt(); @@ -164,6 +186,7 @@ rt_test! { assert_eq!(out, "ZOMG"); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_many_from_block_on() { use tokio::sync::mpsc; @@ -214,6 +237,7 @@ rt_test! { } } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_many_from_task() { use tokio::sync::mpsc; @@ -226,14 +250,6 @@ rt_test! { tokio::spawn(async move { let (done_tx, mut done_rx) = mpsc::unbounded_channel(); - /* - for _ in 0..100 { - tokio::spawn(async move { }); - } - - tokio::task::yield_now().await; - */ - let mut txs = (0..ITER) .map(|i| { let (tx, rx) = oneshot::channel(); @@ -275,6 +291,31 @@ rt_test! { } #[test] + fn spawn_one_from_block_on_called_on_handle() { + let rt = rt(); + let (tx, rx) = oneshot::channel(); + + #[allow(clippy::async_yields_async)] + let handle = rt.handle().block_on(async { + tokio::spawn(async move { + tx.send("ZOMG").unwrap(); + "DONE" + }) + }); + + let out = rt.block_on(async { + let msg = assert_ok!(rx.await); + + let out = assert_ok!(handle.await); + assert_eq!(out, "DONE"); + + msg + }); + + assert_eq!(out, "ZOMG"); + } + + #[test] fn spawn_await_chain() { let rt = rt(); @@ -329,6 +370,7 @@ rt_test! { assert_eq!(out, "ZOMG"); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn complete_block_on_under_load() { let rt = rt(); @@ -352,6 +394,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn complete_task_under_load() { let rt = rt(); @@ -381,6 +424,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_from_other_thread_idle() { let rt = rt(); @@ -401,6 +445,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_from_other_thread_under_load() { let rt = rt(); @@ -461,6 +506,7 @@ rt_test! { assert!(now.elapsed() >= dur); } + #[cfg(not(target_os="wasi"))] // Wasi does not support bind #[test] fn block_on_socket() { let rt = rt(); @@ -481,6 +527,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_from_blocking() { let rt = rt(); @@ -496,6 +543,7 @@ rt_test! { assert_eq!(out, "hello") } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_blocking_from_blocking() { let rt = rt(); @@ -511,6 +559,7 @@ rt_test! { assert_eq!(out, "hello") } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn sleep_from_blocking() { let rt = rt(); @@ -531,6 +580,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support bind #[test] fn socket_from_blocking() { let rt = rt(); @@ -554,6 +604,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn always_active_parker() { // This test it to show that we will always have @@ -600,6 +651,7 @@ rt_test! { // concern. There also isn't a great/obvious solution to take. For now, the // test is disabled. #[cfg(not(windows))] + #[cfg(not(target_os="wasi"))] // Wasi does not support bind or threads fn io_driver_called_when_under_load() { let rt = rt(); @@ -607,7 +659,12 @@ rt_test! { for _ in 0..100 { rt.spawn(async { loop { - tokio::task::yield_now().await; + // Don't use Tokio's `yield_now()` to avoid special defer + // logic. + futures::future::poll_fn::<(), _>(|cx| { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + }).await; } }); } @@ -635,6 +692,72 @@ rt_test! { }); } + /// Tests that yielded tasks are not scheduled until **after** resource + /// drivers are polled. + /// + /// Note: we may have to delete this test as it is not necessarily reliable. + /// The OS does not guarantee when I/O events are delivered, so there may be + /// more yields than anticipated. + #[test] + #[cfg(not(target_os="wasi"))] + fn yield_defers_until_park() { + use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; + use std::sync::Barrier; + + let rt = rt(); + + let flag = Arc::new(AtomicBool::new(false)); + let barrier = Arc::new(Barrier::new(NUM_WORKERS)); + + rt.block_on(async { + // Make sure other workers cannot steal tasks + #[allow(clippy::reversed_empty_ranges)] + for _ in 0..(NUM_WORKERS-1) { + let flag = flag.clone(); + let barrier = barrier.clone(); + + tokio::spawn(async move { + barrier.wait(); + + while !flag.load(SeqCst) { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + }); + } + + barrier.wait(); + + tokio::spawn(async move { + // Create a TCP litener + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + tokio::join!( + async { + // Done blocking intentionally + let _socket = std::net::TcpStream::connect(addr).unwrap(); + + // Yield until connected + let mut cnt = 0; + while !flag.load(SeqCst){ + tokio::task::yield_now().await; + cnt += 1; + + if cnt >= 10 { + panic!("yielded too many times; TODO: delete this test?"); + } + } + }, + async { + let _ = listener.accept().await.unwrap(); + flag.store(true, SeqCst); + } + ); + }).await.unwrap(); + }); + } + + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn client_server_block_on() { let rt = rt(); @@ -646,6 +769,7 @@ rt_test! { assert_err!(rx.try_recv()); } + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support threads or panic recovery")] #[test] #[cfg(not(target_os = "android"))] fn panic_in_task() { @@ -675,11 +799,13 @@ rt_test! { #[test] #[should_panic] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] fn panic_in_block_on() { let rt = rt(); rt.block_on(async { panic!() }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads async fn yield_once() { let mut yielded = false; poll_fn(|cx| { @@ -749,7 +875,11 @@ rt_test! { #[test] fn wake_while_rt_is_dropping() { - use tokio::task; + use tokio::sync::Barrier; + use core::sync::atomic::{AtomicBool, Ordering}; + + let drop_triggered = Arc::new(AtomicBool::new(false)); + let set_drop_triggered = drop_triggered.clone(); struct OnDrop<F: FnMut()>(F); @@ -763,17 +893,21 @@ rt_test! { let (tx2, rx2) = oneshot::channel(); let (tx3, rx3) = oneshot::channel(); - let rt = rt(); + let barrier = Arc::new(Barrier::new(4)); + let barrier1 = barrier.clone(); + let barrier2 = barrier.clone(); + let barrier3 = barrier.clone(); - let h1 = rt.clone(); + let rt = rt(); rt.spawn(async move { // Ensure a waker gets stored in oneshot 1. - let _ = rx1.await; + let _ = tokio::join!(rx1, barrier1.wait()); tx3.send(()).unwrap(); }); rt.spawn(async move { + let h1 = tokio::runtime::Handle::current(); // When this task is dropped, we'll be "closing remotes". // We spawn a new task that owns the `tx1`, to move its Drop // out of here. @@ -786,36 +920,40 @@ rt_test! { h1.spawn(async move { tx1.send(()).unwrap(); }); + // Just a sanity check that this entire thing actually happened + set_drop_triggered.store(true, Ordering::Relaxed); }); - let _ = rx2.await; + let _ = tokio::join!(rx2, barrier2.wait()); }); rt.spawn(async move { - let _ = rx3.await; + let _ = tokio::join!(rx3, barrier3.wait()); // We'll never get here, but once task 3 drops, this will // force task 2 to re-schedule since it's waiting on oneshot 2. tx2.send(()).unwrap(); }); - // Tick the loop - rt.block_on(async { - task::yield_now().await; - }); + // Wait until every oneshot channel has been polled. + rt.block_on(barrier.wait()); // Drop the rt drop(rt); + + // Make sure that the spawn actually happened + assert!(drop_triggered.load(Ordering::Relaxed)); } + #[cfg(not(target_os="wasi"))] // Wasi doesn't support UDP or bind() #[test] fn io_notify_while_shutting_down() { - use std::net::Ipv6Addr; + use tokio::net::UdpSocket; use std::sync::Arc; for _ in 1..10 { let runtime = rt(); runtime.block_on(async { - let socket = UdpSocket::bind((Ipv6Addr::LOCALHOST, 0)).await.unwrap(); + let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); let addr = socket.local_addr().unwrap(); let send_half = Arc::new(socket); let recv_half = send_half.clone(); @@ -841,6 +979,7 @@ rt_test! { } } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn shutdown_timeout() { let (tx, rx) = oneshot::channel(); @@ -858,6 +997,7 @@ rt_test! { Arc::try_unwrap(runtime).unwrap().shutdown_timeout(Duration::from_millis(100)); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn shutdown_timeout_0() { let runtime = rt(); @@ -889,6 +1029,7 @@ rt_test! { // See https://github.com/rust-lang/rust/issues/74875 #[test] #[cfg(not(windows))] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support threads")] fn runtime_in_thread_local() { use std::cell::RefCell; use std::thread; @@ -908,6 +1049,7 @@ rt_test! { }).join().unwrap(); } + #[cfg(not(target_os="wasi"))] // Wasi does not support bind async fn client_server(tx: mpsc::Sender<()>) { let server = assert_ok!(TcpListener::bind("127.0.0.1:0").await); @@ -932,6 +1074,7 @@ rt_test! { tx.send(()).unwrap(); } + #[cfg(not(tokio_wasi))] // Wasi does not support bind #[test] fn local_set_block_on_socket() { let rt = rt(); @@ -953,6 +1096,7 @@ rt_test! { }); } + #[cfg(not(tokio_wasi))] // Wasi does not support bind #[test] fn local_set_client_server_block_on() { let rt = rt(); @@ -966,6 +1110,7 @@ rt_test! { assert_err!(rx.try_recv()); } + #[cfg(not(tokio_wasi))] // Wasi does not support bind async fn client_server_local(tx: mpsc::Sender<()>) { let server = assert_ok!(TcpListener::bind("127.0.0.1:0").await); @@ -993,22 +1138,22 @@ rt_test! { #[test] fn coop() { use std::task::Poll::Ready; + use tokio::sync::mpsc; let rt = rt(); rt.block_on(async { - // Create a bunch of tasks - let mut tasks = (0..1_000).map(|_| { - tokio::spawn(async { }) - }).collect::<Vec<_>>(); + let (send, mut recv) = mpsc::unbounded_channel(); - // Hope that all the tasks complete... - time::sleep(Duration::from_millis(100)).await; + // Send a bunch of messages. + for _ in 0..1_000 { + send.send(()).unwrap(); + } poll_fn(|cx| { - // At least one task should not be ready - for task in &mut tasks { - if Pin::new(task).poll(cx).is_pending() { + // At least one response should return pending. + for _ in 0..1_000 { + if recv.poll_recv(cx).is_pending() { return Ready(()); } } @@ -1021,22 +1166,22 @@ rt_test! { #[test] fn coop_unconstrained() { use std::task::Poll::Ready; + use tokio::sync::mpsc; let rt = rt(); rt.block_on(async { - // Create a bunch of tasks - let mut tasks = (0..1_000).map(|_| { - tokio::spawn(async { }) - }).collect::<Vec<_>>(); + let (send, mut recv) = mpsc::unbounded_channel(); - // Hope that all the tasks complete... - time::sleep(Duration::from_millis(100)).await; + // Send a bunch of messages. + for _ in 0..1_000 { + send.send(()).unwrap(); + } tokio::task::unconstrained(poll_fn(|cx| { - // All the tasks should be ready - for task in &mut tasks { - assert!(Pin::new(task).poll(cx).is_ready()); + // All the responses should be ready. + for _ in 0..1_000 { + assert_eq!(recv.poll_recv(cx), Poll::Ready(Some(()))); } Ready(()) @@ -1044,6 +1189,31 @@ rt_test! { }); } + #[cfg(tokio_unstable)] + #[test] + fn coop_consume_budget() { + let rt = rt(); + + rt.block_on(async { + poll_fn(|cx| { + let counter = Arc::new(std::sync::Mutex::new(0)); + let counter_clone = Arc::clone(&counter); + let mut worker = Box::pin(async move { + // Consume the budget until a yield happens + for _ in 0..1000 { + *counter.lock().unwrap() += 1; + task::consume_budget().await + } + }); + // Assert that the worker was yielded and it didn't manage + // to finish the whole work (assuming the total budget of 128) + assert!(Pin::new(&mut worker).poll(cx).is_pending()); + assert!(*counter_clone.lock().unwrap() < 1000); + std::task::Poll::Ready(()) + }).await; + }); + } + // Tests that the "next task" scheduler optimization is not able to starve // other tasks. #[test] @@ -1061,7 +1231,7 @@ rt_test! { let (spawned_tx, mut spawned_rx) = mpsc::unbounded_channel(); let mut tasks = vec![]; - // Spawn a bunch of tasks that ping ping between each other to + // Spawn a bunch of tasks that ping-pong between each other to // saturate the runtime. for _ in 0..NUM { let (tx1, mut rx1) = mpsc::unbounded_channel(); diff --git a/tests/rt_handle_block_on.rs b/tests/rt_handle_block_on.rs index 17878c8..5ec783e 100644 --- a/tests/rt_handle_block_on.rs +++ b/tests/rt_handle_block_on.rs @@ -10,9 +10,10 @@ use std::time::Duration; use tokio::runtime::{Handle, Runtime}; use tokio::sync::mpsc; -use tokio::task::spawn_blocking; -use tokio::{fs, net, time}; +#[cfg(not(tokio_wasi))] +use tokio::{net, time}; +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads macro_rules! multi_threaded_rt_test { ($($t:tt)*) => { mod threaded_scheduler_4_threads_only { @@ -45,6 +46,7 @@ macro_rules! multi_threaded_rt_test { } } +#[cfg(not(tokio_wasi))] macro_rules! rt_test { ($($t:tt)*) => { mod current_thread_scheduler { @@ -124,7 +126,9 @@ fn unbounded_mpsc_channel() { }) } +#[cfg(not(tokio_wasi))] // Wasi doesn't support file operations or bind rt_test! { + use tokio::fs; // ==== spawn blocking futures ====== #[test] @@ -135,7 +139,7 @@ rt_test! { let contents = Handle::current() .block_on(fs::read_to_string("Cargo.toml")) .unwrap(); - assert!(contents.contains("Cargo.toml")); + assert!(contents.contains("https://tokio.rs")); } #[test] @@ -156,6 +160,7 @@ rt_test! { #[test] fn basic_spawn_blocking() { + use tokio::task::spawn_blocking; let rt = rt(); let _enter = rt.enter(); @@ -171,6 +176,7 @@ rt_test! { #[test] fn spawn_blocking_after_shutdown_fails() { + use tokio::task::spawn_blocking; let rt = rt(); let _enter = rt.enter(); rt.shutdown_timeout(Duration::from_secs(1000)); @@ -187,6 +193,7 @@ rt_test! { #[test] fn spawn_blocking_started_before_shutdown_continues() { + use tokio::task::spawn_blocking; let rt = rt(); let _enter = rt.enter(); @@ -412,6 +419,7 @@ rt_test! { } } +#[cfg(not(tokio_wasi))] multi_threaded_rt_test! { #[cfg(unix)] #[test] @@ -474,6 +482,7 @@ multi_threaded_rt_test! { // ==== utils ====== /// Create a new multi threaded runtime +#[cfg(not(tokio_wasi))] fn new_multi_thread(n: usize) -> Runtime { tokio::runtime::Builder::new_multi_thread() .worker_threads(n) @@ -496,37 +505,30 @@ where F: Fn(), { { - println!("current thread runtime"); - let rt = new_current_thread(); let _enter = rt.enter(); f(); - println!("current thread runtime after shutdown"); rt.shutdown_timeout(Duration::from_secs(1000)); f(); } + #[cfg(not(tokio_wasi))] { - println!("multi thread (1 thread) runtime"); - let rt = new_multi_thread(1); let _enter = rt.enter(); f(); - println!("multi thread runtime after shutdown"); rt.shutdown_timeout(Duration::from_secs(1000)); f(); } + #[cfg(not(tokio_wasi))] { - println!("multi thread (4 threads) runtime"); - let rt = new_multi_thread(4); let _enter = rt.enter(); f(); - println!("multi thread runtime after shutdown"); rt.shutdown_timeout(Duration::from_secs(1000)); f(); } diff --git a/tests/rt_metrics.rs b/tests/rt_metrics.rs new file mode 100644 index 0000000..fdb2fb5 --- /dev/null +++ b/tests/rt_metrics.rs @@ -0,0 +1,481 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", tokio_unstable, not(tokio_wasi)))] + +use std::sync::{Arc, Mutex}; + +use tokio::runtime::Runtime; +use tokio::time::{self, Duration}; + +#[test] +fn num_workers() { + let rt = current_thread(); + assert_eq!(1, rt.metrics().num_workers()); + + let rt = threaded(); + assert_eq!(2, rt.metrics().num_workers()); +} + +#[test] +fn num_blocking_threads() { + let rt = current_thread(); + assert_eq!(0, rt.metrics().num_blocking_threads()); + let _ = rt.block_on(rt.spawn_blocking(move || {})); + assert_eq!(1, rt.metrics().num_blocking_threads()); +} + +#[test] +fn num_idle_blocking_threads() { + let rt = current_thread(); + assert_eq!(0, rt.metrics().num_idle_blocking_threads()); + let _ = rt.block_on(rt.spawn_blocking(move || {})); + rt.block_on(async { + time::sleep(Duration::from_millis(5)).await; + }); + + // We need to wait until the blocking thread has become idle. Usually 5ms is + // enough for this to happen, but not always. When it isn't enough, sleep + // for another second. We don't always wait for a whole second since we want + // the test suite to finish quickly. + // + // Note that the timeout for idle threads to be killed is 10 seconds. + if 0 == rt.metrics().num_idle_blocking_threads() { + rt.block_on(async { + time::sleep(Duration::from_secs(1)).await; + }); + } + + assert_eq!(1, rt.metrics().num_idle_blocking_threads()); +} + +#[test] +fn blocking_queue_depth() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + + assert_eq!(0, rt.metrics().blocking_queue_depth()); + + let ready = Arc::new(Mutex::new(())); + let guard = ready.lock().unwrap(); + + let ready_cloned = ready.clone(); + let wait_until_ready = move || { + let _unused = ready_cloned.lock().unwrap(); + }; + + let h1 = rt.spawn_blocking(wait_until_ready.clone()); + let h2 = rt.spawn_blocking(wait_until_ready); + assert!(rt.metrics().blocking_queue_depth() > 0); + + drop(guard); + + let _ = rt.block_on(h1); + let _ = rt.block_on(h2); + + assert_eq!(0, rt.metrics().blocking_queue_depth()); +} + +#[test] +fn remote_schedule_count() { + use std::thread; + + let rt = current_thread(); + let handle = rt.handle().clone(); + let task = thread::spawn(move || { + handle.spawn(async { + // DO nothing + }) + }) + .join() + .unwrap(); + + rt.block_on(task).unwrap(); + + assert_eq!(1, rt.metrics().remote_schedule_count()); + + let rt = threaded(); + let handle = rt.handle().clone(); + let task = thread::spawn(move || { + handle.spawn(async { + // DO nothing + }) + }) + .join() + .unwrap(); + + rt.block_on(task).unwrap(); + + assert_eq!(1, rt.metrics().remote_schedule_count()); +} + +#[test] +fn worker_park_count() { + let rt = current_thread(); + let metrics = rt.metrics(); + rt.block_on(async { + time::sleep(Duration::from_millis(1)).await; + }); + drop(rt); + assert!(1 <= metrics.worker_park_count(0)); + + let rt = threaded(); + let metrics = rt.metrics(); + rt.block_on(async { + time::sleep(Duration::from_millis(1)).await; + }); + drop(rt); + assert!(1 <= metrics.worker_park_count(0)); + assert!(1 <= metrics.worker_park_count(1)); +} + +#[test] +fn worker_noop_count() { + // There isn't really a great way to generate no-op parks as they happen as + // false-positive events under concurrency. + + let rt = current_thread(); + let metrics = rt.metrics(); + rt.block_on(async { + time::sleep(Duration::from_millis(1)).await; + }); + drop(rt); + assert!(0 < metrics.worker_noop_count(0)); + + let rt = threaded(); + let metrics = rt.metrics(); + rt.block_on(async { + time::sleep(Duration::from_millis(1)).await; + }); + drop(rt); + assert!(0 < metrics.worker_noop_count(0)); + assert!(0 < metrics.worker_noop_count(1)); +} + +#[test] +fn worker_steal_count() { + // This metric only applies to the multi-threaded runtime. + // + // We use a blocking channel to backup one worker thread. + use std::sync::mpsc::channel; + + let rt = threaded(); + let metrics = rt.metrics(); + + rt.block_on(async { + let (tx, rx) = channel(); + + // Move to the runtime. + tokio::spawn(async move { + // Spawn the task that sends to the channel + tokio::spawn(async move { + tx.send(()).unwrap(); + }); + + // Spawn a task that bumps the previous task out of the "next + // scheduled" slot. + tokio::spawn(async {}); + + // Blocking receive on the channel. + rx.recv().unwrap(); + }) + .await + .unwrap(); + }); + + drop(rt); + + let n: u64 = (0..metrics.num_workers()) + .map(|i| metrics.worker_steal_count(i)) + .sum(); + + assert_eq!(1, n); +} + +#[test] +fn worker_poll_count() { + const N: u64 = 5; + + let rt = current_thread(); + let metrics = rt.metrics(); + rt.block_on(async { + for _ in 0..N { + tokio::spawn(async {}).await.unwrap(); + } + }); + drop(rt); + assert_eq!(N, metrics.worker_poll_count(0)); + + let rt = threaded(); + let metrics = rt.metrics(); + rt.block_on(async { + for _ in 0..N { + tokio::spawn(async {}).await.unwrap(); + } + }); + drop(rt); + // Account for the `block_on` task + let n = (0..metrics.num_workers()) + .map(|i| metrics.worker_poll_count(i)) + .sum(); + + assert_eq!(N, n); +} + +#[test] +fn worker_total_busy_duration() { + const N: usize = 5; + + let zero = Duration::from_millis(0); + + let rt = current_thread(); + let metrics = rt.metrics(); + + rt.block_on(async { + for _ in 0..N { + tokio::spawn(async { + tokio::task::yield_now().await; + }) + .await + .unwrap(); + } + }); + + drop(rt); + + assert!(zero < metrics.worker_total_busy_duration(0)); + + let rt = threaded(); + let metrics = rt.metrics(); + + rt.block_on(async { + for _ in 0..N { + tokio::spawn(async { + tokio::task::yield_now().await; + }) + .await + .unwrap(); + } + }); + + drop(rt); + + for i in 0..metrics.num_workers() { + assert!(zero < metrics.worker_total_busy_duration(i)); + } +} + +#[test] +fn worker_local_schedule_count() { + let rt = current_thread(); + let metrics = rt.metrics(); + rt.block_on(async { + tokio::spawn(async {}).await.unwrap(); + }); + drop(rt); + + assert_eq!(1, metrics.worker_local_schedule_count(0)); + assert_eq!(0, metrics.remote_schedule_count()); + + let rt = threaded(); + let metrics = rt.metrics(); + rt.block_on(async { + // Move to the runtime + tokio::spawn(async { + tokio::spawn(async {}).await.unwrap(); + }) + .await + .unwrap(); + }); + drop(rt); + + let n: u64 = (0..metrics.num_workers()) + .map(|i| metrics.worker_local_schedule_count(i)) + .sum(); + + assert_eq!(2, n); + assert_eq!(1, metrics.remote_schedule_count()); +} + +#[test] +fn worker_overflow_count() { + // Only applies to the threaded worker + let rt = threaded(); + let metrics = rt.metrics(); + rt.block_on(async { + // Move to the runtime + tokio::spawn(async { + let (tx1, rx1) = std::sync::mpsc::channel(); + let (tx2, rx2) = std::sync::mpsc::channel(); + + // First, we need to block the other worker until all tasks have + // been spawned. + tokio::spawn(async move { + tx1.send(()).unwrap(); + rx2.recv().unwrap(); + }); + + // Bump the next-run spawn + tokio::spawn(async {}); + + rx1.recv().unwrap(); + + // Spawn many tasks + for _ in 0..300 { + tokio::spawn(async {}); + } + + tx2.send(()).unwrap(); + }) + .await + .unwrap(); + }); + drop(rt); + + let n: u64 = (0..metrics.num_workers()) + .map(|i| metrics.worker_overflow_count(i)) + .sum(); + + assert_eq!(1, n); +} + +#[test] +fn injection_queue_depth() { + use std::thread; + + let rt = current_thread(); + let handle = rt.handle().clone(); + let metrics = rt.metrics(); + + thread::spawn(move || { + handle.spawn(async {}); + }) + .join() + .unwrap(); + + assert_eq!(1, metrics.injection_queue_depth()); + + let rt = threaded(); + let handle = rt.handle().clone(); + let metrics = rt.metrics(); + + // First we need to block the runtime workers + let (tx1, rx1) = std::sync::mpsc::channel(); + let (tx2, rx2) = std::sync::mpsc::channel(); + + rt.spawn(async move { rx1.recv().unwrap() }); + rt.spawn(async move { rx2.recv().unwrap() }); + + thread::spawn(move || { + handle.spawn(async {}); + }) + .join() + .unwrap(); + + let n = metrics.injection_queue_depth(); + assert!(1 <= n, "{}", n); + assert!(3 >= n, "{}", n); + + tx1.send(()).unwrap(); + tx2.send(()).unwrap(); +} + +#[test] +fn worker_local_queue_depth() { + const N: usize = 100; + + let rt = current_thread(); + let metrics = rt.metrics(); + rt.block_on(async { + for _ in 0..N { + tokio::spawn(async {}); + } + + assert_eq!(N, metrics.worker_local_queue_depth(0)); + }); + + let rt = threaded(); + let metrics = rt.metrics(); + rt.block_on(async move { + // Move to the runtime + tokio::spawn(async move { + let (tx1, rx1) = std::sync::mpsc::channel(); + let (tx2, rx2) = std::sync::mpsc::channel(); + + // First, we need to block the other worker until all tasks have + // been spawned. + tokio::spawn(async move { + tx1.send(()).unwrap(); + rx2.recv().unwrap(); + }); + + // Bump the next-run spawn + tokio::spawn(async {}); + + rx1.recv().unwrap(); + + // Spawn some tasks + for _ in 0..100 { + tokio::spawn(async {}); + } + + let n: usize = (0..metrics.num_workers()) + .map(|i| metrics.worker_local_queue_depth(i)) + .sum(); + + assert_eq!(n, N); + + tx2.send(()).unwrap(); + }) + .await + .unwrap(); + }); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn io_driver_fd_count() { + let rt = current_thread(); + let metrics = rt.metrics(); + + assert_eq!(metrics.io_driver_fd_registered_count(), 0); + + let stream = tokio::net::TcpStream::connect("google.com:80"); + let stream = rt.block_on(async move { stream.await.unwrap() }); + + assert_eq!(metrics.io_driver_fd_registered_count(), 1); + assert_eq!(metrics.io_driver_fd_deregistered_count(), 0); + + drop(stream); + + assert_eq!(metrics.io_driver_fd_deregistered_count(), 1); + assert_eq!(metrics.io_driver_fd_registered_count(), 1); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn io_driver_ready_count() { + let rt = current_thread(); + let metrics = rt.metrics(); + + let stream = tokio::net::TcpStream::connect("google.com:80"); + let _stream = rt.block_on(async move { stream.await.unwrap() }); + + assert_eq!(metrics.io_driver_ready_count(), 1); +} + +fn current_thread() -> Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +fn threaded() -> Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() +} diff --git a/tests/rt_panic.rs b/tests/rt_panic.rs new file mode 100644 index 0000000..f9a684f --- /dev/null +++ b/tests/rt_panic.rs @@ -0,0 +1,77 @@ +#![warn(rust_2018_idioms)] +#![cfg(feature = "full")] +#![cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery + +use futures::future; +use std::error::Error; +use tokio::runtime::{Builder, Handle, Runtime}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn current_handle_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _ = Handle::current(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn into_panic_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(move || { + let rt = current_thread(); + rt.block_on(async { + let handle = tokio::spawn(future::pending::<()>()); + + handle.abort(); + + let err = handle.await.unwrap_err(); + assert!(!&err.is_panic()); + + let _ = err.into_panic(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn builder_worker_threads_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _ = Builder::new_multi_thread().worker_threads(0).build(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn builder_max_blocking_threads_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _ = Builder::new_multi_thread().max_blocking_threads(0).build(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +fn current_thread() -> Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} diff --git a/tests/rt_threaded.rs b/tests/rt_threaded.rs index 5f047a7..c598418 100644 --- a/tests/rt_threaded.rs +++ b/tests/rt_threaded.rs @@ -1,9 +1,9 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use tokio::runtime::{self, Runtime}; +use tokio::runtime; use tokio::sync::oneshot; use tokio_test::{assert_err, assert_ok}; @@ -15,6 +15,15 @@ use std::sync::atomic::Ordering::Relaxed; use std::sync::{mpsc, Arc, Mutex}; use std::task::{Context, Poll, Waker}; +macro_rules! cfg_metrics { + ($($t:tt)*) => { + #[cfg(tokio_unstable)] + { + $( $t )* + } + } +} + #[test] fn single_thread() { // No panic when starting a runtime w/ a single thread @@ -56,6 +65,38 @@ fn many_oneshot_futures() { } #[test] +fn spawn_two() { + let rt = rt(); + + let out = rt.block_on(async { + let (tx, rx) = oneshot::channel(); + + tokio::spawn(async move { + tokio::spawn(async move { + tx.send("ZOMG").unwrap(); + }); + }); + + assert_ok!(rx.await) + }); + + assert_eq!(out, "ZOMG"); + + cfg_metrics! { + let metrics = rt.metrics(); + drop(rt); + assert_eq!(1, metrics.remote_schedule_count()); + + let mut local = 0; + for i in 0..metrics.num_workers() { + local += metrics.worker_local_schedule_count(i); + } + + assert_eq!(1, local); + } +} + +#[test] fn many_multishot_futures() { const CHAIN: usize = 200; const CYCLES: usize = 5; @@ -374,6 +415,32 @@ fn coop_and_block_in_place() { }); } +#[test] +fn yield_after_block_in_place() { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + + rt.block_on(async { + tokio::spawn(async move { + // Block in place then enter a new runtime + tokio::task::block_in_place(|| { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(async {}); + }); + + // Yield, then complete + tokio::task::yield_now().await; + }) + .await + .unwrap() + }); +} + // Testing this does not panic #[test] fn max_blocking_threads() { @@ -439,9 +506,7 @@ fn wake_during_shutdown() { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { let me = Pin::into_inner(self); let mut lock = me.shared.lock().unwrap(); - println!("poll {}", me.put_waker); if me.put_waker { - println!("putting"); lock.waker = Some(cx.waker().clone()); } Poll::Pending @@ -450,13 +515,11 @@ fn wake_during_shutdown() { impl Drop for MyFuture { fn drop(&mut self) { - println!("drop {} start", self.put_waker); let mut lock = self.shared.lock().unwrap(); if !self.put_waker { lock.waker.take().unwrap().wake(); } drop(lock); - println!("drop {} stop", self.put_waker); } } @@ -498,6 +561,30 @@ async fn test_block_in_place4() { tokio::task::block_in_place(|| {}); } -fn rt() -> Runtime { - Runtime::new().unwrap() +fn rt() -> runtime::Runtime { + runtime::Runtime::new().unwrap() +} + +#[cfg(tokio_unstable)] +mod unstable { + use super::*; + + #[test] + fn test_disable_lifo_slot() { + let rt = runtime::Builder::new_multi_thread() + .disable_lifo_slot() + .worker_threads(2) + .build() + .unwrap(); + + rt.block_on(async { + tokio::spawn(async { + // Spawn another task and block the thread until completion. If the LIFO slot + // is used then the test doesn't complete. + futures::executor::block_on(tokio::spawn(async {})).unwrap(); + }) + .await + .unwrap(); + }) + } } diff --git a/tests/signal_no_rt.rs b/tests/signal_no_rt.rs index b0f32b2..ffcc665 100644 --- a/tests/signal_no_rt.rs +++ b/tests/signal_no_rt.rs @@ -4,6 +4,7 @@ use tokio::signal::unix::{signal, SignalKind}; +#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] #[test] #[should_panic] fn no_runtime_panics_creating_signals() { diff --git a/tests/signal_panic.rs b/tests/signal_panic.rs new file mode 100644 index 0000000..ce1ec3e --- /dev/null +++ b/tests/signal_panic.rs @@ -0,0 +1,29 @@ +#![warn(rust_2018_idioms)] +#![cfg(feature = "full")] +#![cfg(unix)] + +use std::error::Error; +use tokio::runtime::Builder; +use tokio::signal::unix::{signal, SignalKind}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn signal_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = Builder::new_current_thread().build().unwrap(); + + rt.block_on(async { + let kind = SignalKind::from_raw(-1); + let _ = signal(kind); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/tests/support/leaked_buffers.rs b/tests/support/leaked_buffers.rs new file mode 100644 index 0000000..a6079fb --- /dev/null +++ b/tests/support/leaked_buffers.rs @@ -0,0 +1,26 @@ +/// Can create buffers of arbitrary lifetime. +/// Frees created buffers when dropped. +/// +/// This struct is of course unsafe and the fact that +/// it must outlive the created slices has to be ensured by +/// the programmer. +/// +/// Used at certain test scenarios as a safer version of +/// Vec::leak, to satisfy the address sanitizer. +pub struct LeakedBuffers { + leaked_vecs: Vec<Box<[u8]>>, +} + +impl LeakedBuffers { + pub fn new() -> Self { + Self { + leaked_vecs: vec![], + } + } + pub unsafe fn create<'a>(&mut self, size: usize) -> &'a mut [u8] { + let new_mem = vec![0u8; size].into_boxed_slice(); + self.leaked_vecs.push(new_mem); + let new_mem = self.leaked_vecs.last_mut().unwrap(); + std::slice::from_raw_parts_mut(new_mem.as_mut_ptr(), new_mem.len()) + } +} diff --git a/tests/support/panic.rs b/tests/support/panic.rs new file mode 100644 index 0000000..df2f59d --- /dev/null +++ b/tests/support/panic.rs @@ -0,0 +1,34 @@ +use std::panic; +use std::sync::{Arc, Mutex}; + +pub fn test_panic<Func: FnOnce() + panic::UnwindSafe>(func: Func) -> Option<String> { + static PANIC_MUTEX: Mutex<()> = Mutex::new(()); + + { + let _guard = PANIC_MUTEX.lock(); + let panic_file: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None)); + + let prev_hook = panic::take_hook(); + { + let panic_file = panic_file.clone(); + panic::set_hook(Box::new(move |panic_info| { + let panic_location = panic_info.location().unwrap(); + panic_file + .lock() + .unwrap() + .clone_from(&Some(panic_location.file().to_string())); + })); + } + + let result = panic::catch_unwind(func); + // Return to the previously set panic hook (maybe default) so that we get nice error + // messages in the tests. + panic::set_hook(prev_hook); + + if result.is_err() { + panic_file.lock().unwrap().clone() + } else { + None + } + } +} diff --git a/tests/sync_barrier.rs b/tests/sync_barrier.rs index f280fe8..2001c35 100644 --- a/tests/sync_barrier.rs +++ b/tests/sync_barrier.rs @@ -1,6 +1,9 @@ #![allow(clippy::unnecessary_operation)] #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::Barrier; diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs index 9ef7927..cd66924 100644 --- a/tests/sync_broadcast.rs +++ b/tests/sync_broadcast.rs @@ -2,6 +2,9 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; + use tokio::sync::broadcast; use tokio_test::task; use tokio_test::{ @@ -44,7 +47,7 @@ macro_rules! assert_closed { ($e:expr) => { match assert_err!($e) { broadcast::error::TryRecvError::Closed => {} - _ => panic!("did not lag"), + _ => panic!("is not closed"), } }; } @@ -273,12 +276,14 @@ fn send_no_rx() { #[test] #[should_panic] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn zero_capacity() { broadcast::channel::<()>(0); } #[test] #[should_panic] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn capacity_too_big() { use std::usize; @@ -286,7 +291,8 @@ fn capacity_too_big() { } #[test] -#[cfg(not(target_os = "android"))] +#[cfg(panic = "unwind")] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn panic_in_clone() { use std::panic::{self, AssertUnwindSafe}; @@ -452,6 +458,132 @@ fn lagging_receiver_recovers_after_wrap_open() { assert_empty!(rx); } +#[test] +fn receiver_len_with_lagged() { + let (tx, mut rx) = broadcast::channel(3); + + tx.send(10).unwrap(); + tx.send(20).unwrap(); + tx.send(30).unwrap(); + tx.send(40).unwrap(); + + assert_eq!(rx.len(), 4); + assert_eq!(assert_recv!(rx), 10); + + tx.send(50).unwrap(); + tx.send(60).unwrap(); + + assert_eq!(rx.len(), 5); + assert_lagged!(rx.try_recv(), 1); +} + fn is_closed(err: broadcast::error::RecvError) -> bool { matches!(err, broadcast::error::RecvError::Closed) } + +#[test] +fn resubscribe_points_to_tail() { + let (tx, mut rx) = broadcast::channel(3); + tx.send(1).unwrap(); + + let mut rx_resub = rx.resubscribe(); + + // verify we're one behind at the start + assert_empty!(rx_resub); + assert_eq!(assert_recv!(rx), 1); + + // verify we do not affect rx + tx.send(2).unwrap(); + assert_eq!(assert_recv!(rx_resub), 2); + tx.send(3).unwrap(); + assert_eq!(assert_recv!(rx), 2); + assert_eq!(assert_recv!(rx), 3); + assert_empty!(rx); + + assert_eq!(assert_recv!(rx_resub), 3); + assert_empty!(rx_resub); +} + +#[test] +fn resubscribe_lagged() { + let (tx, mut rx) = broadcast::channel(1); + tx.send(1).unwrap(); + tx.send(2).unwrap(); + + let mut rx_resub = rx.resubscribe(); + assert_lagged!(rx.try_recv(), 1); + assert_empty!(rx_resub); + + assert_eq!(assert_recv!(rx), 2); + assert_empty!(rx); + assert_empty!(rx_resub); +} + +#[test] +fn resubscribe_to_closed_channel() { + let (tx, rx) = tokio::sync::broadcast::channel::<u32>(2); + drop(tx); + + let mut rx_resub = rx.resubscribe(); + assert_closed!(rx_resub.try_recv()); +} + +#[test] +fn sender_len() { + let (tx, mut rx1) = broadcast::channel(4); + let mut rx2 = tx.subscribe(); + + assert_eq!(tx.len(), 0); + assert!(tx.is_empty()); + + tx.send(1).unwrap(); + tx.send(2).unwrap(); + tx.send(3).unwrap(); + + assert_eq!(tx.len(), 3); + assert!(!tx.is_empty()); + + assert_recv!(rx1); + assert_recv!(rx1); + + assert_eq!(tx.len(), 3); + assert!(!tx.is_empty()); + + assert_recv!(rx2); + + assert_eq!(tx.len(), 2); + assert!(!tx.is_empty()); + + tx.send(4).unwrap(); + tx.send(5).unwrap(); + tx.send(6).unwrap(); + + assert_eq!(tx.len(), 4); + assert!(!tx.is_empty()); +} + +#[test] +#[cfg(not(tokio_wasm_not_wasi))] +fn sender_len_random() { + use rand::Rng; + + let (tx, mut rx1) = broadcast::channel(16); + let mut rx2 = tx.subscribe(); + + for _ in 0..1000 { + match rand::thread_rng().gen_range(0..4) { + 0 => { + let _ = rx1.try_recv(); + } + 1 => { + let _ = rx2.try_recv(); + } + _ => { + tx.send(0).unwrap(); + } + } + + let expected_len = usize::min(usize::max(rx1.len(), rx2.len()), 16); + assert_eq!(tx.len(), expected_len); + } +} diff --git a/tests/sync_errors.rs b/tests/sync_errors.rs index 66e8f0c..bf0a6cf 100644 --- a/tests/sync_errors.rs +++ b/tests/sync_errors.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; fn is_error<T: std::error::Error + Send + Sync>() {} diff --git a/tests/sync_mpsc.rs b/tests/sync_mpsc.rs index 1947d26..6e87096 100644 --- a/tests/sync_mpsc.rs +++ b/tests/sync_mpsc.rs @@ -1,18 +1,21 @@ #![allow(clippy::redundant_clone)] #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] -use std::thread; -use tokio::runtime::Runtime; -use tokio::sync::mpsc; -use tokio::sync::mpsc::error::{TryRecvError, TrySendError}; -use tokio_test::task; -use tokio_test::{ - assert_err, assert_ok, assert_pending, assert_ready, assert_ready_err, assert_ready_ok, -}; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; +use std::fmt; use std::sync::Arc; +use tokio::sync::mpsc; +use tokio::sync::mpsc::error::{TryRecvError, TrySendError}; +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; +use tokio_test::*; +#[cfg(not(tokio_wasm))] mod support { pub(crate) mod mpsc_stream; } @@ -21,7 +24,7 @@ trait AssertSend: Send {} impl AssertSend for mpsc::Sender<i32> {} impl AssertSend for mpsc::Receiver<i32> {} -#[tokio::test] +#[maybe_tokio_test] async fn send_recv_with_buffer() { let (tx, mut rx) = mpsc::channel::<i32>(16); @@ -46,6 +49,7 @@ async fn send_recv_with_buffer() { } #[tokio::test] +#[cfg(feature = "full")] async fn reserve_disarm() { let (tx, mut rx) = mpsc::channel::<i32>(2); let tx1 = tx.clone(); @@ -58,10 +62,10 @@ async fn reserve_disarm() { let permit2 = assert_ok!(tx2.reserve().await); // But a third should not be ready - let mut r3 = task::spawn(tx3.reserve()); + let mut r3 = tokio_test::task::spawn(tx3.reserve()); assert_pending!(r3.poll()); - let mut r4 = task::spawn(tx4.reserve()); + let mut r4 = tokio_test::task::spawn(tx4.reserve()); assert_pending!(r4.poll()); // Using one of the reserved slots should allow a new handle to become ready @@ -78,11 +82,12 @@ async fn reserve_disarm() { drop(permit2); assert!(r4.is_woken()); - let mut r1 = task::spawn(tx1.reserve()); + let mut r1 = tokio_test::task::spawn(tx1.reserve()); assert_pending!(r1.poll()); } #[tokio::test] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads async fn send_recv_stream_with_buffer() { use tokio_stream::StreamExt; @@ -100,6 +105,7 @@ async fn send_recv_stream_with_buffer() { } #[tokio::test] +#[cfg(feature = "full")] async fn async_send_recv_with_buffer() { let (tx, mut rx) = mpsc::channel(16); @@ -114,10 +120,11 @@ async fn async_send_recv_with_buffer() { } #[tokio::test] +#[cfg(feature = "full")] async fn start_send_past_cap() { use std::future::Future; - let mut t1 = task::spawn(()); + let mut t1 = tokio_test::task::spawn(()); let (tx1, mut rx) = mpsc::channel(1); let tx2 = tx1.clone(); @@ -128,7 +135,7 @@ async fn start_send_past_cap() { t1.enter(|cx, _| assert_pending!(r1.as_mut().poll(cx))); { - let mut r2 = task::spawn(tx2.reserve()); + let mut r2 = tokio_test::task::spawn(tx2.reserve()); assert_pending!(r2.poll()); drop(r1); @@ -147,11 +154,12 @@ async fn start_send_past_cap() { #[test] #[should_panic] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn buffer_gteq_one() { mpsc::channel::<i32>(0); } -#[tokio::test] +#[maybe_tokio_test] async fn send_recv_unbounded() { let (tx, mut rx) = mpsc::unbounded_channel::<i32>(); @@ -168,6 +176,7 @@ async fn send_recv_unbounded() { } #[tokio::test] +#[cfg(feature = "full")] async fn async_send_recv_unbounded() { let (tx, mut rx) = mpsc::unbounded_channel(); @@ -182,6 +191,7 @@ async fn async_send_recv_unbounded() { } #[tokio::test] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads async fn send_recv_stream_unbounded() { use tokio_stream::StreamExt; @@ -199,32 +209,32 @@ async fn send_recv_stream_unbounded() { assert_eq!(None, rx.next().await); } -#[tokio::test] +#[maybe_tokio_test] async fn no_t_bounds_buffer() { struct NoImpls; let (tx, mut rx) = mpsc::channel(100); // sender should be Debug even though T isn't Debug - println!("{:?}", tx); + is_debug(&tx); // same with Receiver - println!("{:?}", rx); + is_debug(&rx); // and sender should be Clone even though T isn't Clone assert!(tx.clone().try_send(NoImpls).is_ok()); assert!(rx.recv().await.is_some()); } -#[tokio::test] +#[maybe_tokio_test] async fn no_t_bounds_unbounded() { struct NoImpls; let (tx, mut rx) = mpsc::unbounded_channel(); // sender should be Debug even though T isn't Debug - println!("{:?}", tx); + is_debug(&tx); // same with Receiver - println!("{:?}", rx); + is_debug(&rx); // and sender should be Clone even though T isn't Clone assert!(tx.clone().send(NoImpls).is_ok()); @@ -232,6 +242,7 @@ async fn no_t_bounds_unbounded() { } #[tokio::test] +#[cfg(feature = "full")] async fn send_recv_buffer_limited() { let (tx, mut rx) = mpsc::channel::<i32>(1); @@ -242,7 +253,7 @@ async fn send_recv_buffer_limited() { p1.send(1); // Not ready - let mut p2 = task::spawn(tx.reserve()); + let mut p2 = tokio_test::task::spawn(tx.reserve()); assert_pending!(p2.poll()); // Take the value @@ -261,7 +272,7 @@ async fn send_recv_buffer_limited() { assert!(rx.recv().await.is_some()); } -#[tokio::test] +#[maybe_tokio_test] async fn recv_close_gets_none_idle() { let (tx, mut rx) = mpsc::channel::<i32>(10); @@ -273,12 +284,13 @@ async fn recv_close_gets_none_idle() { } #[tokio::test] +#[cfg(feature = "full")] async fn recv_close_gets_none_reserved() { let (tx1, mut rx) = mpsc::channel::<i32>(1); let tx2 = tx1.clone(); let permit1 = assert_ok!(tx1.reserve().await); - let mut permit2 = task::spawn(tx2.reserve()); + let mut permit2 = tokio_test::task::spawn(tx2.reserve()); assert_pending!(permit2.poll()); rx.close(); @@ -287,7 +299,7 @@ async fn recv_close_gets_none_reserved() { assert_ready_err!(permit2.poll()); { - let mut recv = task::spawn(rx.recv()); + let mut recv = tokio_test::task::spawn(rx.recv()); assert_pending!(recv.poll()); permit1.send(123); @@ -300,13 +312,13 @@ async fn recv_close_gets_none_reserved() { assert!(rx.recv().await.is_none()); } -#[tokio::test] +#[maybe_tokio_test] async fn tx_close_gets_none() { let (_, mut rx) = mpsc::channel::<i32>(10); assert!(rx.recv().await.is_none()); } -#[tokio::test] +#[maybe_tokio_test] async fn try_send_fail() { let (tx, mut rx) = mpsc::channel(1); @@ -327,7 +339,7 @@ async fn try_send_fail() { assert!(rx.recv().await.is_none()); } -#[tokio::test] +#[maybe_tokio_test] async fn try_send_fail_with_try_recv() { let (tx, mut rx) = mpsc::channel(1); @@ -348,7 +360,7 @@ async fn try_send_fail_with_try_recv() { assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); } -#[tokio::test] +#[maybe_tokio_test] async fn try_reserve_fails() { let (tx, mut rx) = mpsc::channel(1); @@ -372,6 +384,7 @@ async fn try_reserve_fails() { } #[tokio::test] +#[cfg(feature = "full")] async fn drop_permit_releases_permit() { // poll_ready reserves capacity, ensure that the capacity is released if tx // is dropped w/o sending a value. @@ -380,7 +393,7 @@ async fn drop_permit_releases_permit() { let permit = assert_ok!(tx1.reserve().await); - let mut reserve2 = task::spawn(tx2.reserve()); + let mut reserve2 = tokio_test::task::spawn(tx2.reserve()); assert_pending!(reserve2.poll()); drop(permit); @@ -389,7 +402,7 @@ async fn drop_permit_releases_permit() { assert_ready_ok!(reserve2.poll()); } -#[tokio::test] +#[maybe_tokio_test] async fn dropping_rx_closes_channel() { let (tx, rx) = mpsc::channel(100); @@ -439,48 +452,57 @@ fn unconsumed_messages_are_dropped() { } #[test] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads fn blocking_recv() { let (tx, mut rx) = mpsc::channel::<u8>(1); - let sync_code = thread::spawn(move || { + let sync_code = std::thread::spawn(move || { assert_eq!(Some(10), rx.blocking_recv()); }); - Runtime::new().unwrap().block_on(async move { - let _ = tx.send(10).await; - }); + tokio::runtime::Runtime::new() + .unwrap() + .block_on(async move { + let _ = tx.send(10).await; + }); sync_code.join().unwrap() } #[tokio::test] #[should_panic] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding async fn blocking_recv_async() { let (_tx, mut rx) = mpsc::channel::<()>(1); let _ = rx.blocking_recv(); } #[test] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads fn blocking_send() { let (tx, mut rx) = mpsc::channel::<u8>(1); - let sync_code = thread::spawn(move || { + let sync_code = std::thread::spawn(move || { tx.blocking_send(10).unwrap(); }); - Runtime::new().unwrap().block_on(async move { - assert_eq!(Some(10), rx.recv().await); - }); + tokio::runtime::Runtime::new() + .unwrap() + .block_on(async move { + assert_eq!(Some(10), rx.recv().await); + }); sync_code.join().unwrap() } #[tokio::test] #[should_panic] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding async fn blocking_send_async() { let (tx, _rx) = mpsc::channel::<()>(1); let _ = tx.blocking_send(()); } #[tokio::test] +#[cfg(feature = "full")] async fn ready_close_cancel_bounded() { let (tx, mut rx) = mpsc::channel::<()>(100); let _tx2 = tx.clone(); @@ -489,7 +511,7 @@ async fn ready_close_cancel_bounded() { rx.close(); - let mut recv = task::spawn(rx.recv()); + let mut recv = tokio_test::task::spawn(rx.recv()); assert_pending!(recv.poll()); drop(permit); @@ -500,13 +522,14 @@ async fn ready_close_cancel_bounded() { } #[tokio::test] +#[cfg(feature = "full")] async fn permit_available_not_acquired_close() { let (tx1, mut rx) = mpsc::channel::<()>(1); let tx2 = tx1.clone(); let permit1 = assert_ok!(tx1.reserve().await); - let mut permit2 = task::spawn(tx2.reserve()); + let mut permit2 = tokio_test::task::spawn(tx2.reserve()); assert_pending!(permit2.poll()); rx.close(); @@ -597,3 +620,60 @@ fn try_recv_close_while_empty_unbounded() { drop(tx); assert_eq!(Err(TryRecvError::Disconnected), rx.try_recv()); } + +#[tokio::test(start_paused = true)] +#[cfg(feature = "full")] +async fn recv_timeout() { + use tokio::sync::mpsc::error::SendTimeoutError::{Closed, Timeout}; + use tokio::time::Duration; + + let (tx, rx) = mpsc::channel(5); + + assert_eq!(tx.send_timeout(10, Duration::from_secs(1)).await, Ok(())); + assert_eq!(tx.send_timeout(20, Duration::from_secs(1)).await, Ok(())); + assert_eq!(tx.send_timeout(30, Duration::from_secs(1)).await, Ok(())); + assert_eq!(tx.send_timeout(40, Duration::from_secs(1)).await, Ok(())); + assert_eq!(tx.send_timeout(50, Duration::from_secs(1)).await, Ok(())); + assert_eq!( + tx.send_timeout(60, Duration::from_secs(1)).await, + Err(Timeout(60)) + ); + + drop(rx); + assert_eq!( + tx.send_timeout(70, Duration::from_secs(1)).await, + Err(Closed(70)) + ); +} + +#[test] +#[should_panic = "there is no reactor running, must be called from the context of a Tokio 1.x runtime"] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +fn recv_timeout_panic() { + use futures::future::FutureExt; + use tokio::time::Duration; + + let (tx, _rx) = mpsc::channel(5); + tx.send_timeout(10, Duration::from_secs(1)).now_or_never(); +} + +// Tests that channel `capacity` changes and `max_capacity` stays the same +#[tokio::test] +async fn test_tx_capacity() { + let (tx, _rx) = mpsc::channel::<()>(10); + // both capacities are same before + assert_eq!(tx.capacity(), 10); + assert_eq!(tx.max_capacity(), 10); + + let _permit = tx.reserve().await.unwrap(); + // after reserve, only capacity should drop by one + assert_eq!(tx.capacity(), 9); + assert_eq!(tx.max_capacity(), 10); + + tx.send(()).await.unwrap(); + // after send, capacity should drop by one again + assert_eq!(tx.capacity(), 8); + assert_eq!(tx.max_capacity(), 10); +} + +fn is_debug<T: fmt::Debug>(_: &T) {} diff --git a/tests/sync_mpsc_weak.rs b/tests/sync_mpsc_weak.rs new file mode 100644 index 0000000..0fdfc00 --- /dev/null +++ b/tests/sync_mpsc_weak.rs @@ -0,0 +1,513 @@ +#![allow(clippy::redundant_clone)] +#![warn(rust_2018_idioms)] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; + +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::{Acquire, Release}; +use tokio::sync::mpsc::{self, channel, unbounded_channel}; +use tokio::sync::oneshot; + +#[tokio::test] +async fn weak_sender() { + let (tx, mut rx) = channel(11); + + let tx_weak = tokio::spawn(async move { + let tx_weak = tx.clone().downgrade(); + + for i in 0..10 { + if tx.send(i).await.is_err() { + return None; + } + } + + let tx2 = tx_weak + .upgrade() + .expect("expected to be able to upgrade tx_weak"); + let _ = tx2.send(20).await; + let tx_weak = tx2.downgrade(); + + Some(tx_weak) + }) + .await + .unwrap(); + + for i in 0..12 { + let recvd = rx.recv().await; + + match recvd { + Some(msg) => { + if i == 10 { + assert_eq!(msg, 20); + } + } + None => { + assert_eq!(i, 11); + break; + } + } + } + + let tx_weak = tx_weak.unwrap(); + let upgraded = tx_weak.upgrade(); + assert!(upgraded.is_none()); +} + +#[tokio::test] +async fn actor_weak_sender() { + pub struct MyActor { + receiver: mpsc::Receiver<ActorMessage>, + sender: mpsc::WeakSender<ActorMessage>, + next_id: u32, + pub received_self_msg: bool, + } + + enum ActorMessage { + GetUniqueId { respond_to: oneshot::Sender<u32> }, + SelfMessage {}, + } + + impl MyActor { + fn new( + receiver: mpsc::Receiver<ActorMessage>, + sender: mpsc::WeakSender<ActorMessage>, + ) -> Self { + MyActor { + receiver, + sender, + next_id: 0, + received_self_msg: false, + } + } + + fn handle_message(&mut self, msg: ActorMessage) { + match msg { + ActorMessage::GetUniqueId { respond_to } => { + self.next_id += 1; + + // The `let _ =` ignores any errors when sending. + // + // This can happen if the `select!` macro is used + // to cancel waiting for the response. + let _ = respond_to.send(self.next_id); + } + ActorMessage::SelfMessage { .. } => { + self.received_self_msg = true; + } + } + } + + async fn send_message_to_self(&mut self) { + let msg = ActorMessage::SelfMessage {}; + + let sender = self.sender.clone(); + + // cannot move self.sender here + if let Some(sender) = sender.upgrade() { + let _ = sender.send(msg).await; + self.sender = sender.downgrade(); + } + } + + async fn run(&mut self) { + let mut i = 0; + while let Some(msg) = self.receiver.recv().await { + self.handle_message(msg); + + if i == 0 { + self.send_message_to_self().await; + } + + i += 1 + } + + assert!(self.received_self_msg); + } + } + + #[derive(Clone)] + pub struct MyActorHandle { + sender: mpsc::Sender<ActorMessage>, + } + + impl MyActorHandle { + pub fn new() -> (Self, MyActor) { + let (sender, receiver) = mpsc::channel(8); + let actor = MyActor::new(receiver, sender.clone().downgrade()); + + (Self { sender }, actor) + } + + pub async fn get_unique_id(&self) -> u32 { + let (send, recv) = oneshot::channel(); + let msg = ActorMessage::GetUniqueId { respond_to: send }; + + // Ignore send errors. If this send fails, so does the + // recv.await below. There's no reason to check the + // failure twice. + let _ = self.sender.send(msg).await; + recv.await.expect("Actor task has been killed") + } + } + + let (handle, mut actor) = MyActorHandle::new(); + + let actor_handle = tokio::spawn(async move { actor.run().await }); + + let _ = tokio::spawn(async move { + let _ = handle.get_unique_id().await; + drop(handle); + }) + .await; + + let _ = actor_handle.await; +} + +static NUM_DROPPED: AtomicUsize = AtomicUsize::new(0); + +#[derive(Debug)] +struct Msg; + +impl Drop for Msg { + fn drop(&mut self) { + NUM_DROPPED.fetch_add(1, Release); + } +} + +// Tests that no pending messages are put onto the channel after `Rx` was +// dropped. +// +// Note: After the introduction of `WeakSender`, which internally +// used `Arc` and doesn't call a drop of the channel after the last strong +// `Sender` was dropped while more than one `WeakSender` remains, we want to +// ensure that no messages are kept in the channel, which were sent after +// the receiver was dropped. +#[tokio::test] +async fn test_msgs_dropped_on_rx_drop() { + let (tx, mut rx) = mpsc::channel(3); + + tx.send(Msg {}).await.unwrap(); + tx.send(Msg {}).await.unwrap(); + + // This msg will be pending and should be dropped when `rx` is dropped + let sent_fut = tx.send(Msg {}); + + let _ = rx.recv().await.unwrap(); + let _ = rx.recv().await.unwrap(); + + sent_fut.await.unwrap(); + + drop(rx); + + assert_eq!(NUM_DROPPED.load(Acquire), 3); + + // This msg will not be put onto `Tx` list anymore, since `Rx` is closed. + assert!(tx.send(Msg {}).await.is_err()); + + assert_eq!(NUM_DROPPED.load(Acquire), 4); +} + +// Tests that a `WeakSender` is upgradeable when other `Sender`s exist. +#[test] +fn downgrade_upgrade_sender_success() { + let (tx, _rx) = mpsc::channel::<i32>(1); + let weak_tx = tx.downgrade(); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that a `WeakSender` fails to upgrade when no other `Sender` exists. +#[test] +fn downgrade_upgrade_sender_failure() { + let (tx, _rx) = mpsc::channel::<i32>(1); + let weak_tx = tx.downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that a `WeakSender` cannot be upgraded after a `Sender` was dropped, +// which existed at the time of the `downgrade` call. +#[test] +fn downgrade_drop_upgrade() { + let (tx, _rx) = mpsc::channel::<i32>(1); + + // the cloned `Tx` is dropped right away + let weak_tx = tx.clone().downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that we can upgrade a weak sender with an outstanding permit +// but no other strong senders. +#[tokio::test] +async fn downgrade_get_permit_upgrade_no_senders() { + let (tx, _rx) = mpsc::channel::<i32>(1); + let weak_tx = tx.downgrade(); + let _permit = tx.reserve_owned().await.unwrap(); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that you can downgrade and upgrade a sender with an outstanding permit +// but no other senders left. +#[tokio::test] +async fn downgrade_upgrade_get_permit_no_senders() { + let (tx, _rx) = mpsc::channel::<i32>(1); + let tx2 = tx.clone(); + let _permit = tx.reserve_owned().await.unwrap(); + let weak_tx = tx2.downgrade(); + drop(tx2); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that `downgrade` does not change the `tx_count` of the channel. +#[test] +fn test_tx_count_weak_sender() { + let (tx, _rx) = mpsc::channel::<i32>(1); + let tx_weak = tx.downgrade(); + let tx_weak2 = tx.downgrade(); + drop(tx); + + assert!(tx_weak.upgrade().is_none() && tx_weak2.upgrade().is_none()); +} + +#[tokio::test] +async fn weak_unbounded_sender() { + let (tx, mut rx) = unbounded_channel(); + + let tx_weak = tokio::spawn(async move { + let tx_weak = tx.clone().downgrade(); + + for i in 0..10 { + if tx.send(i).is_err() { + return None; + } + } + + let tx2 = tx_weak + .upgrade() + .expect("expected to be able to upgrade tx_weak"); + let _ = tx2.send(20); + let tx_weak = tx2.downgrade(); + + Some(tx_weak) + }) + .await + .unwrap(); + + for i in 0..12 { + let recvd = rx.recv().await; + + match recvd { + Some(msg) => { + if i == 10 { + assert_eq!(msg, 20); + } + } + None => { + assert_eq!(i, 11); + break; + } + } + } + + let tx_weak = tx_weak.unwrap(); + let upgraded = tx_weak.upgrade(); + assert!(upgraded.is_none()); +} + +#[tokio::test] +async fn actor_weak_unbounded_sender() { + pub struct MyActor { + receiver: mpsc::UnboundedReceiver<ActorMessage>, + sender: mpsc::WeakUnboundedSender<ActorMessage>, + next_id: u32, + pub received_self_msg: bool, + } + + enum ActorMessage { + GetUniqueId { respond_to: oneshot::Sender<u32> }, + SelfMessage {}, + } + + impl MyActor { + fn new( + receiver: mpsc::UnboundedReceiver<ActorMessage>, + sender: mpsc::WeakUnboundedSender<ActorMessage>, + ) -> Self { + MyActor { + receiver, + sender, + next_id: 0, + received_self_msg: false, + } + } + + fn handle_message(&mut self, msg: ActorMessage) { + match msg { + ActorMessage::GetUniqueId { respond_to } => { + self.next_id += 1; + + // The `let _ =` ignores any errors when sending. + // + // This can happen if the `select!` macro is used + // to cancel waiting for the response. + let _ = respond_to.send(self.next_id); + } + ActorMessage::SelfMessage { .. } => { + self.received_self_msg = true; + } + } + } + + async fn send_message_to_self(&mut self) { + let msg = ActorMessage::SelfMessage {}; + + let sender = self.sender.clone(); + + // cannot move self.sender here + if let Some(sender) = sender.upgrade() { + let _ = sender.send(msg); + self.sender = sender.downgrade(); + } + } + + async fn run(&mut self) { + let mut i = 0; + while let Some(msg) = self.receiver.recv().await { + self.handle_message(msg); + + if i == 0 { + self.send_message_to_self().await; + } + + i += 1 + } + + assert!(self.received_self_msg); + } + } + + #[derive(Clone)] + pub struct MyActorHandle { + sender: mpsc::UnboundedSender<ActorMessage>, + } + + impl MyActorHandle { + pub fn new() -> (Self, MyActor) { + let (sender, receiver) = mpsc::unbounded_channel(); + let actor = MyActor::new(receiver, sender.clone().downgrade()); + + (Self { sender }, actor) + } + + pub async fn get_unique_id(&self) -> u32 { + let (send, recv) = oneshot::channel(); + let msg = ActorMessage::GetUniqueId { respond_to: send }; + + // Ignore send errors. If this send fails, so does the + // recv.await below. There's no reason to check the + // failure twice. + let _ = self.sender.send(msg); + recv.await.expect("Actor task has been killed") + } + } + + let (handle, mut actor) = MyActorHandle::new(); + + let actor_handle = tokio::spawn(async move { actor.run().await }); + + let _ = tokio::spawn(async move { + let _ = handle.get_unique_id().await; + drop(handle); + }) + .await; + + let _ = actor_handle.await; +} + +static NUM_DROPPED_UNBOUNDED: AtomicUsize = AtomicUsize::new(0); + +#[derive(Debug)] +struct MsgUnbounded; + +impl Drop for MsgUnbounded { + fn drop(&mut self) { + NUM_DROPPED_UNBOUNDED.fetch_add(1, Release); + } +} + +// Tests that no pending messages are put onto the channel after `Rx` was +// dropped. +// +// Note: After the introduction of `UnboundedWeakSender`, which internally +// used `Arc` and doesn't call a drop of the channel after the last strong +// `UnboundedSender` was dropped while more than one `UnboundedWeakSender` +// remains, we want to ensure that no messages are kept in the channel, which +// were sent after the receiver was dropped. +#[tokio::test] +async fn test_msgs_dropped_on_unbounded_rx_drop() { + let (tx, mut rx) = mpsc::unbounded_channel(); + + tx.send(MsgUnbounded {}).unwrap(); + tx.send(MsgUnbounded {}).unwrap(); + + // This msg will be pending and should be dropped when `rx` is dropped + let sent = tx.send(MsgUnbounded {}); + + let _ = rx.recv().await.unwrap(); + let _ = rx.recv().await.unwrap(); + + sent.unwrap(); + + drop(rx); + + assert_eq!(NUM_DROPPED_UNBOUNDED.load(Acquire), 3); + + // This msg will not be put onto `Tx` list anymore, since `Rx` is closed. + assert!(tx.send(MsgUnbounded {}).is_err()); + + assert_eq!(NUM_DROPPED_UNBOUNDED.load(Acquire), 4); +} + +// Tests that an `WeakUnboundedSender` is upgradeable when other +// `UnboundedSender`s exist. +#[test] +fn downgrade_upgrade_unbounded_sender_success() { + let (tx, _rx) = mpsc::unbounded_channel::<i32>(); + let weak_tx = tx.downgrade(); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that a `WeakUnboundedSender` fails to upgrade when no other +// `UnboundedSender` exists. +#[test] +fn downgrade_upgrade_unbounded_sender_failure() { + let (tx, _rx) = mpsc::unbounded_channel::<i32>(); + let weak_tx = tx.downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that an `WeakUnboundedSender` cannot be upgraded after an +// `UnboundedSender` was dropped, which existed at the time of the `downgrade` call. +#[test] +fn downgrade_drop_upgrade_unbounded() { + let (tx, _rx) = mpsc::unbounded_channel::<i32>(); + + // the cloned `Tx` is dropped right away + let weak_tx = tx.clone().downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that `downgrade` does not change the `tx_count` of the channel. +#[test] +fn test_tx_count_weak_unbounded_sender() { + let (tx, _rx) = mpsc::unbounded_channel::<i32>(); + let tx_weak = tx.downgrade(); + let tx_weak2 = tx.downgrade(); + drop(tx); + + assert!(tx_weak.upgrade().is_none() && tx_weak2.upgrade().is_none()); +} diff --git a/tests/sync_mutex.rs b/tests/sync_mutex.rs index 090db94..1e35a55 100644 --- a/tests/sync_mutex.rs +++ b/tests/sync_mutex.rs @@ -1,13 +1,19 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; use tokio::sync::Mutex; -use tokio::time::{interval, timeout}; use tokio_test::task::spawn; use tokio_test::{assert_pending, assert_ready}; use std::sync::Arc; -use std::time::Duration; #[test] fn straight_execution() { @@ -47,7 +53,7 @@ fn readiness() { // But once g unlocks, we can acquire it drop(g); assert!(t2.is_woken()); - assert_ready!(t2.poll()); + let _t2 = assert_ready!(t2.poll()); } /* @@ -82,10 +88,14 @@ fn lock() { } */ -#[tokio::test] /// Ensure a mutex is unlocked if a future holding the lock /// is aborted prematurely. +#[tokio::test] +#[cfg(feature = "full")] async fn aborted_future_1() { + use std::time::Duration; + use tokio::time::{interval, timeout}; + let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0)); { let m2 = m1.clone(); @@ -93,7 +103,7 @@ async fn aborted_future_1() { timeout(Duration::from_millis(1u64), async move { let iv = interval(Duration::from_millis(1000)); tokio::pin!(iv); - m2.lock().await; + let _g = m2.lock().await; iv.as_mut().tick().await; iv.as_mut().tick().await; }) @@ -102,16 +112,20 @@ async fn aborted_future_1() { } // This should succeed as there is no lock left for the mutex. timeout(Duration::from_millis(1u64), async move { - m1.lock().await; + let _g = m1.lock().await; }) .await .expect("Mutex is locked"); } -#[tokio::test] /// This test is similar to `aborted_future_1` but this time the /// aborted future is waiting for the lock. +#[tokio::test] +#[cfg(feature = "full")] async fn aborted_future_2() { + use std::time::Duration; + use tokio::time::timeout; + let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0)); { // Lock mutex @@ -120,7 +134,7 @@ async fn aborted_future_2() { let m2 = m1.clone(); // Try to lock mutex in a future that is aborted prematurely timeout(Duration::from_millis(1u64), async move { - m2.lock().await; + let _g = m2.lock().await; }) .await .unwrap_err(); @@ -128,7 +142,7 @@ async fn aborted_future_2() { } // This should succeed as there is no lock left for the mutex. timeout(Duration::from_millis(1u64), async move { - m1.lock().await; + let _g = m1.lock().await; }) .await .expect("Mutex is locked"); @@ -141,20 +155,20 @@ fn try_lock() { let g1 = m.try_lock(); assert!(g1.is_ok()); let g2 = m.try_lock(); - assert!(!g2.is_ok()); + assert!(g2.is_err()); } let g3 = m.try_lock(); assert!(g3.is_ok()); } -#[tokio::test] +#[maybe_tokio_test] async fn debug_format() { let s = "debug"; let m = Mutex::new(s.to_string()); assert_eq!(format!("{:?}", s), format!("{:?}", m.lock().await)); } -#[tokio::test] +#[maybe_tokio_test] async fn mutex_debug() { let s = "data"; let m = Mutex::new(s.to_string()); diff --git a/tests/sync_mutex_owned.rs b/tests/sync_mutex_owned.rs index 898bf35..ba472fe 100644 --- a/tests/sync_mutex_owned.rs +++ b/tests/sync_mutex_owned.rs @@ -1,13 +1,19 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; use tokio::sync::Mutex; -use tokio::time::{interval, timeout}; use tokio_test::task::spawn; use tokio_test::{assert_pending, assert_ready}; use std::sync::Arc; -use std::time::Duration; #[test] fn straight_execution() { @@ -49,10 +55,14 @@ fn readiness() { assert_ready!(t2.poll()); } -#[tokio::test] /// Ensure a mutex is unlocked if a future holding the lock /// is aborted prematurely. +#[tokio::test] +#[cfg(feature = "full")] async fn aborted_future_1() { + use std::time::Duration; + use tokio::time::{interval, timeout}; + let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0)); { let m2 = m1.clone(); @@ -75,10 +85,14 @@ async fn aborted_future_1() { .expect("Mutex is locked"); } -#[tokio::test] /// This test is similar to `aborted_future_1` but this time the /// aborted future is waiting for the lock. +#[tokio::test] +#[cfg(feature = "full")] async fn aborted_future_2() { + use std::time::Duration; + use tokio::time::timeout; + let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0)); { // Lock mutex @@ -108,13 +122,13 @@ fn try_lock_owned() { let g1 = m.clone().try_lock_owned(); assert!(g1.is_ok()); let g2 = m.clone().try_lock_owned(); - assert!(!g2.is_ok()); + assert!(g2.is_err()); } let g3 = m.try_lock_owned(); assert!(g3.is_ok()); } -#[tokio::test] +#[maybe_tokio_test] async fn debug_format() { let s = "debug"; let m = Arc::new(Mutex::new(s.to_string())); diff --git a/tests/sync_notify.rs b/tests/sync_notify.rs index 6c6620b..aa58039 100644 --- a/tests/sync_notify.rs +++ b/tests/sync_notify.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::Notify; use tokio_test::task::spawn; @@ -151,3 +154,74 @@ fn notify_one_after_dropped_all() { assert_ready!(notified2.poll()); } + +#[test] +fn test_notify_one_not_enabled() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + notify.notify_one(); + assert_ready!(future.poll()); +} + +#[test] +fn test_notify_one_after_enable() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + future.enter(|_, fut| assert!(!fut.enable())); + + notify.notify_one(); + assert_ready!(future.poll()); + future.enter(|_, fut| assert!(fut.enable())); +} + +#[test] +fn test_poll_after_enable() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + future.enter(|_, fut| assert!(!fut.enable())); + assert_pending!(future.poll()); +} + +#[test] +fn test_enable_after_poll() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + assert_pending!(future.poll()); + future.enter(|_, fut| assert!(!fut.enable())); +} + +#[test] +fn test_enable_consumes_permit() { + let notify = Notify::new(); + + // Add a permit. + notify.notify_one(); + + let mut future1 = spawn(notify.notified()); + future1.enter(|_, fut| assert!(fut.enable())); + + let mut future2 = spawn(notify.notified()); + future2.enter(|_, fut| assert!(!fut.enable())); +} + +#[test] +fn test_waker_update() { + use futures::task::noop_waker; + use std::future::Future; + use std::task::Context; + + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + let noop = noop_waker(); + future.enter(|_, fut| assert_pending!(fut.poll(&mut Context::from_waker(&noop)))); + + assert_pending!(future.poll()); + notify.notify_one(); + + assert!(future.is_woken()); +} diff --git a/tests/sync_once_cell.rs b/tests/sync_once_cell.rs index 18eaf93..38dfa7c 100644 --- a/tests/sync_once_cell.rs +++ b/tests/sync_once_cell.rs @@ -4,178 +4,7 @@ use std::mem; use std::ops::Drop; use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::Duration; -use tokio::runtime; -use tokio::sync::{OnceCell, SetError}; -use tokio::time; - -async fn func1() -> u32 { - 5 -} - -async fn func2() -> u32 { - time::sleep(Duration::from_millis(1)).await; - 10 -} - -async fn func_err() -> Result<u32, ()> { - Err(()) -} - -async fn func_ok() -> Result<u32, ()> { - Ok(10) -} - -async fn func_panic() -> u32 { - time::sleep(Duration::from_millis(1)).await; - panic!(); -} - -async fn sleep_and_set() -> u32 { - // Simulate sleep by pausing time and waiting for another thread to - // resume clock when calling `set`, then finding the cell being initialized - // by this call - time::sleep(Duration::from_millis(2)).await; - 5 -} - -async fn advance_time_and_set(cell: &'static OnceCell<u32>, v: u32) -> Result<(), SetError<u32>> { - time::advance(Duration::from_millis(1)).await; - cell.set(v) -} - -#[test] -fn get_or_init() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .start_paused(true) - .build() - .unwrap(); - - static ONCE: OnceCell<u32> = OnceCell::const_new(); - - rt.block_on(async { - let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); - let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await }); - - time::advance(Duration::from_millis(1)).await; - time::resume(); - - let result1 = handle1.await.unwrap(); - let result2 = handle2.await.unwrap(); - - assert_eq!(*result1, 5); - assert_eq!(*result2, 5); - }); -} - -#[test] -fn get_or_init_panic() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - static ONCE: OnceCell<u32> = OnceCell::const_new(); - - rt.block_on(async { - time::pause(); - - let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); - let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await }); - - time::advance(Duration::from_millis(1)).await; - - let result1 = handle1.await.unwrap(); - let result2 = handle2.await.unwrap(); - - assert_eq!(*result1, 5); - assert_eq!(*result2, 5); - }); -} - -#[test] -fn set_and_get() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - static ONCE: OnceCell<u32> = OnceCell::const_new(); - - rt.block_on(async { - let _ = rt.spawn(async { ONCE.set(5) }).await; - let value = ONCE.get().unwrap(); - assert_eq!(*value, 5); - }); -} - -#[test] -fn get_uninit() { - static ONCE: OnceCell<u32> = OnceCell::const_new(); - let uninit = ONCE.get(); - assert!(uninit.is_none()); -} - -#[test] -fn set_twice() { - static ONCE: OnceCell<u32> = OnceCell::const_new(); - - let first = ONCE.set(5); - assert_eq!(first, Ok(())); - let second = ONCE.set(6); - assert!(second.err().unwrap().is_already_init_err()); -} - -#[test] -fn set_while_initializing() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - static ONCE: OnceCell<u32> = OnceCell::const_new(); - - rt.block_on(async { - time::pause(); - - let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await }); - let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await }); - - time::advance(Duration::from_millis(2)).await; - - let result1 = handle1.await.unwrap(); - let result2 = handle2.await.unwrap(); - - assert_eq!(*result1, 5); - assert!(result2.err().unwrap().is_initializing_err()); - }); -} - -#[test] -fn get_or_try_init() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .start_paused(true) - .build() - .unwrap(); - - static ONCE: OnceCell<u32> = OnceCell::const_new(); - - rt.block_on(async { - let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await }); - let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await }); - - time::advance(Duration::from_millis(1)).await; - time::resume(); - - let result1 = handle1.await.unwrap(); - assert!(result1.is_err()); - - let result2 = handle2.await.unwrap(); - assert_eq!(*result2.unwrap(), 10); - }); -} +use tokio::sync::OnceCell; #[test] fn drop_cell() { @@ -272,3 +101,185 @@ fn from() { let cell = OnceCell::from(2); assert_eq!(*cell.get().unwrap(), 2); } + +#[cfg(feature = "parking_lot")] +mod parking_lot { + use super::*; + + use tokio::runtime; + use tokio::sync::SetError; + use tokio::time; + + use std::time::Duration; + + async fn func1() -> u32 { + 5 + } + + async fn func2() -> u32 { + time::sleep(Duration::from_millis(1)).await; + 10 + } + + async fn func_err() -> Result<u32, ()> { + Err(()) + } + + async fn func_ok() -> Result<u32, ()> { + Ok(10) + } + + async fn func_panic() -> u32 { + time::sleep(Duration::from_millis(1)).await; + panic!(); + } + + async fn sleep_and_set() -> u32 { + // Simulate sleep by pausing time and waiting for another thread to + // resume clock when calling `set`, then finding the cell being initialized + // by this call + time::sleep(Duration::from_millis(2)).await; + 5 + } + + async fn advance_time_and_set( + cell: &'static OnceCell<u32>, + v: u32, + ) -> Result<(), SetError<u32>> { + time::advance(Duration::from_millis(1)).await; + cell.set(v) + } + + #[test] + fn get_or_init() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .unwrap(); + + static ONCE: OnceCell<u32> = OnceCell::const_new(); + + rt.block_on(async { + let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); + let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await }); + + time::advance(Duration::from_millis(1)).await; + time::resume(); + + let result1 = handle1.await.unwrap(); + let result2 = handle2.await.unwrap(); + + assert_eq!(*result1, 5); + assert_eq!(*result2, 5); + }); + } + + #[test] + fn get_or_init_panic() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + static ONCE: OnceCell<u32> = OnceCell::const_new(); + + rt.block_on(async { + time::pause(); + + let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); + let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await }); + + time::advance(Duration::from_millis(1)).await; + + let result1 = handle1.await.unwrap(); + let result2 = handle2.await.unwrap(); + + assert_eq!(*result1, 5); + assert_eq!(*result2, 5); + }); + } + + #[test] + fn set_and_get() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + static ONCE: OnceCell<u32> = OnceCell::const_new(); + + rt.block_on(async { + let _ = rt.spawn(async { ONCE.set(5) }).await; + let value = ONCE.get().unwrap(); + assert_eq!(*value, 5); + }); + } + + #[test] + fn get_uninit() { + static ONCE: OnceCell<u32> = OnceCell::const_new(); + let uninit = ONCE.get(); + assert!(uninit.is_none()); + } + + #[test] + fn set_twice() { + static ONCE: OnceCell<u32> = OnceCell::const_new(); + + let first = ONCE.set(5); + assert_eq!(first, Ok(())); + let second = ONCE.set(6); + assert!(second.err().unwrap().is_already_init_err()); + } + + #[test] + fn set_while_initializing() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + static ONCE: OnceCell<u32> = OnceCell::const_new(); + + rt.block_on(async { + time::pause(); + + let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await }); + let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await }); + + time::advance(Duration::from_millis(2)).await; + + let result1 = handle1.await.unwrap(); + let result2 = handle2.await.unwrap(); + + assert_eq!(*result1, 5); + assert!(result2.err().unwrap().is_initializing_err()); + }); + } + + #[test] + fn get_or_try_init() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .unwrap(); + + static ONCE: OnceCell<u32> = OnceCell::const_new(); + + rt.block_on(async { + let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await }); + let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await }); + + time::advance(Duration::from_millis(1)).await; + time::resume(); + + let result1 = handle1.await.unwrap(); + assert!(result1.is_err()); + + let result2 = handle2.await.unwrap(); + assert_eq!(*result2.unwrap(), 10); + }); + } +} diff --git a/tests/sync_oneshot.rs b/tests/sync_oneshot.rs index 1aab810..15b6923 100644 --- a/tests/sync_oneshot.rs +++ b/tests/sync_oneshot.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; use tokio::sync::oneshot; use tokio::sync::oneshot::error::TryRecvError; @@ -40,7 +48,7 @@ fn send_recv() { assert_eq!(val, 1); } -#[tokio::test] +#[maybe_tokio_test] async fn async_send_recv() { let (tx, rx) = oneshot::channel(); @@ -86,6 +94,7 @@ fn close_rx() { } #[tokio::test] +#[cfg(feature = "full")] async fn async_rx_closed() { let (mut tx, rx) = oneshot::channel::<()>(); @@ -170,6 +179,7 @@ fn explicit_close_try_recv() { #[test] #[should_panic] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn close_try_recv_poll() { let (_tx, rx) = oneshot::channel::<i32>(); let mut rx = task::spawn(rx); @@ -203,6 +213,18 @@ fn try_recv_after_completion() { } #[test] +fn try_recv_after_completion_await() { + let (tx, rx) = oneshot::channel::<i32>(); + let mut rx = task::spawn(rx); + + tx.send(17).unwrap(); + + assert_eq!(Ok(17), assert_ready!(rx.poll())); + assert_eq!(Err(TryRecvError::Closed), rx.try_recv()); + rx.close(); +} + +#[test] fn drops_tasks() { let (mut tx, mut rx) = oneshot::channel::<i32>(); let mut tx_task = task::spawn(()); diff --git a/tests/sync_panic.rs b/tests/sync_panic.rs new file mode 100644 index 0000000..6c23664 --- /dev/null +++ b/tests/sync_panic.rs @@ -0,0 +1,197 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] + +use std::{error::Error, sync::Arc}; +use tokio::{ + runtime::{Builder, Runtime}, + sync::{broadcast, mpsc, oneshot, Mutex, RwLock, Semaphore}, +}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn broadcast_channel_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let (_, _) = broadcast::channel::<u32>(0); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mutex_blocking_lock_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let mutex = Mutex::new(5_u32); + let _g = mutex.blocking_lock(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn oneshot_blocking_recv_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let (_tx, rx) = oneshot::channel::<u8>(); + let _ = rx.blocking_recv(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn rwlock_with_max_readers_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _ = RwLock::<u8>::with_max_readers(0, (u32::MAX >> 3) + 1); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn rwlock_blocking_read_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let lock = RwLock::<u8>::new(0); + let _ = lock.blocking_read(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn rwlock_blocking_write_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let lock = RwLock::<u8>::new(0); + let _ = lock.blocking_write(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_bounded_channel_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let (_, _) = mpsc::channel::<u8>(0); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_bounded_receiver_blocking_recv_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + let (_tx, mut rx) = mpsc::channel::<u8>(1); + rt.block_on(async { + let _ = rx.blocking_recv(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_bounded_sender_blocking_send_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + let (tx, _rx) = mpsc::channel::<u8>(1); + rt.block_on(async { + let _ = tx.blocking_send(3); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_unbounded_receiver_blocking_recv_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + let (_tx, mut rx) = mpsc::unbounded_channel::<u8>(); + rt.block_on(async { + let _ = rx.blocking_recv(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn semaphore_merge_unrelated_owned_permits() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let sem1 = Arc::new(Semaphore::new(42)); + let sem2 = Arc::new(Semaphore::new(42)); + let mut p1 = sem1.try_acquire_owned().unwrap(); + let p2 = sem2.try_acquire_owned().unwrap(); + p1.merge(p2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn semaphore_merge_unrelated_permits() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let sem1 = Semaphore::new(42); + let sem2 = Semaphore::new(42); + let mut p1 = sem1.try_acquire().unwrap(); + let p2 = sem2.try_acquire().unwrap(); + p1.merge(p2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +fn current_thread() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} diff --git a/tests/sync_rwlock.rs b/tests/sync_rwlock.rs index 7d05086..948ec13 100644 --- a/tests/sync_rwlock.rs +++ b/tests/sync_rwlock.rs @@ -1,13 +1,19 @@ #![warn(rust_2018_idioms)] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; + +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; -use std::sync::Arc; use std::task::Poll; use futures::future::FutureExt; -use futures::stream; -use futures::stream::StreamExt; -use tokio::sync::{Barrier, RwLock}; +use tokio::sync::RwLock; use tokio_test::task::spawn; use tokio_test::{assert_pending, assert_ready}; @@ -25,7 +31,7 @@ fn read_shared() { let mut t1 = spawn(rwlock.read()); let _g1 = assert_ready!(t1.poll()); let mut t2 = spawn(rwlock.read()); - assert_ready!(t2.poll()); + let _g2 = assert_ready!(t2.poll()); } // When there is an active shared owner, exclusive access should not be possible @@ -69,7 +75,7 @@ fn exhaust_reading() { let g2 = reads.pop().unwrap(); drop(g2); assert!(t1.is_woken()); - assert_ready!(t1.poll()); + let _g1 = assert_ready!(t1.poll()); } // When there is an active exclusive owner, subsequent exclusive access should not be possible @@ -94,7 +100,7 @@ fn write_shared_drop() { assert_pending!(t2.poll()); drop(g1); assert!(t2.is_woken()); - assert_ready!(t2.poll()); + let _g2 = assert_ready!(t2.poll()); } // when there is an active shared owner, and exclusive access is triggered, @@ -106,7 +112,7 @@ fn write_read_shared_pending() { let _g1 = assert_ready!(t1.poll()); let mut t2 = spawn(rwlock.read()); - assert_ready!(t2.poll()); + let _g2 = assert_ready!(t2.poll()); let mut t3 = spawn(rwlock.write()); assert_pending!(t3.poll()); @@ -131,11 +137,11 @@ fn write_read_shared_drop_pending() { drop(t2); assert!(t3.is_woken()); - assert_ready!(t3.poll()); + let _t3 = assert_ready!(t3.poll()); } // Acquire an RwLock nonexclusively by a single task -#[tokio::test] +#[maybe_tokio_test] async fn read_uncontested() { let rwlock = RwLock::new(100); let result = *rwlock.read().await; @@ -144,7 +150,7 @@ async fn read_uncontested() { } // Acquire an uncontested RwLock in exclusive mode -#[tokio::test] +#[maybe_tokio_test] async fn write_uncontested() { let rwlock = RwLock::new(100); let mut result = rwlock.write().await; @@ -153,7 +159,7 @@ async fn write_uncontested() { } // RwLocks should be acquired in the order that their Futures are waited upon. -#[tokio::test] +#[maybe_tokio_test] async fn write_order() { let rwlock = RwLock::<Vec<u32>>::new(vec![]); let fut2 = rwlock.write().map(|mut guard| guard.push(2)); @@ -166,8 +172,13 @@ async fn write_order() { } // A single RwLock is contested by tasks in multiple threads +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn multithreaded() { + use futures::stream::{self, StreamExt}; + use std::sync::Arc; + use tokio::sync::Barrier; + let barrier = Arc::new(Barrier::new(5)); let rwlock = Arc::new(RwLock::<u32>::new(0)); let rwclone1 = rwlock.clone(); @@ -236,7 +247,7 @@ async fn multithreaded() { assert_eq!(*g, 17_000); } -#[tokio::test] +#[maybe_tokio_test] async fn try_write() { let lock = RwLock::new(0); let read_guard = lock.read().await; diff --git a/tests/sync_semaphore.rs b/tests/sync_semaphore.rs index a33b878..3d47ed0 100644 --- a/tests/sync_semaphore.rs +++ b/tests/sync_semaphore.rs @@ -1,4 +1,7 @@ -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::Arc; use tokio::sync::Semaphore; @@ -23,6 +26,7 @@ fn try_acquire() { } #[tokio::test] +#[cfg(feature = "full")] async fn acquire() { let sem = Arc::new(Semaphore::new(1)); let p1 = sem.try_acquire().unwrap(); @@ -35,6 +39,7 @@ async fn acquire() { } #[tokio::test] +#[cfg(feature = "full")] async fn add_permits() { let sem = Arc::new(Semaphore::new(0)); let sem_clone = sem.clone(); @@ -58,8 +63,34 @@ fn forget() { assert!(sem.try_acquire().is_err()); } +#[test] +fn merge() { + let sem = Arc::new(Semaphore::new(3)); + { + let mut p1 = sem.try_acquire().unwrap(); + assert_eq!(sem.available_permits(), 2); + let p2 = sem.try_acquire_many(2).unwrap(); + assert_eq!(sem.available_permits(), 0); + p1.merge(p2); + assert_eq!(sem.available_permits(), 0); + } + assert_eq!(sem.available_permits(), 3); +} + +#[test] +#[cfg(not(tokio_wasm))] // No stack unwinding on wasm targets +#[should_panic] +fn merge_unrelated_permits() { + let sem1 = Arc::new(Semaphore::new(3)); + let sem2 = Arc::new(Semaphore::new(3)); + let mut p1 = sem1.try_acquire().unwrap(); + let p2 = sem2.try_acquire().unwrap(); + p1.merge(p2); +} + #[tokio::test] -async fn stresstest() { +#[cfg(feature = "full")] +async fn stress_test() { let sem = Arc::new(Semaphore::new(5)); let mut join_handles = Vec::new(); for _ in 0..1000 { @@ -83,13 +114,37 @@ async fn stresstest() { #[test] fn add_max_amount_permits() { let s = tokio::sync::Semaphore::new(0); - s.add_permits(usize::MAX >> 3); - assert_eq!(s.available_permits(), usize::MAX >> 3); + s.add_permits(tokio::sync::Semaphore::MAX_PERMITS); + assert_eq!(s.available_permits(), tokio::sync::Semaphore::MAX_PERMITS); } +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding #[test] #[should_panic] -fn add_more_than_max_amount_permits() { +fn add_more_than_max_amount_permits1() { let s = tokio::sync::Semaphore::new(1); - s.add_permits(usize::MAX >> 3); + s.add_permits(tokio::sync::Semaphore::MAX_PERMITS); +} + +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +#[test] +#[should_panic] +fn add_more_than_max_amount_permits2() { + let s = Semaphore::new(Semaphore::MAX_PERMITS - 1); + s.add_permits(1); + s.add_permits(1); +} + +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +#[test] +#[should_panic] +fn panic_when_exceeds_maxpermits() { + let _ = Semaphore::new(Semaphore::MAX_PERMITS + 1); +} + +#[test] +fn no_panic_at_maxpermits() { + let _ = Semaphore::new(Semaphore::MAX_PERMITS); + let s = Semaphore::new(Semaphore::MAX_PERMITS - 1); + s.add_permits(1); } diff --git a/tests/sync_semaphore_owned.rs b/tests/sync_semaphore_owned.rs index 478c3a3..f694576 100644 --- a/tests/sync_semaphore_owned.rs +++ b/tests/sync_semaphore_owned.rs @@ -1,4 +1,7 @@ -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::Arc; use tokio::sync::Semaphore; @@ -33,6 +36,7 @@ fn try_acquire_many() { } #[tokio::test] +#[cfg(feature = "full")] async fn acquire() { let sem = Arc::new(Semaphore::new(1)); let p1 = sem.clone().try_acquire_owned().unwrap(); @@ -45,6 +49,7 @@ async fn acquire() { } #[tokio::test] +#[cfg(feature = "full")] async fn acquire_many() { let semaphore = Arc::new(Semaphore::new(42)); let permit32 = semaphore.clone().try_acquire_many_owned(32).unwrap(); @@ -60,6 +65,7 @@ async fn acquire_many() { } #[tokio::test] +#[cfg(feature = "full")] async fn add_permits() { let sem = Arc::new(Semaphore::new(0)); let sem_clone = sem.clone(); @@ -83,8 +89,34 @@ fn forget() { assert!(sem.try_acquire_owned().is_err()); } +#[test] +fn merge() { + let sem = Arc::new(Semaphore::new(3)); + { + let mut p1 = sem.clone().try_acquire_owned().unwrap(); + assert_eq!(sem.available_permits(), 2); + let p2 = sem.clone().try_acquire_many_owned(2).unwrap(); + assert_eq!(sem.available_permits(), 0); + p1.merge(p2); + assert_eq!(sem.available_permits(), 0); + } + assert_eq!(sem.available_permits(), 3); +} + +#[test] +#[cfg(not(tokio_wasm))] // No stack unwinding on wasm targets +#[should_panic] +fn merge_unrelated_permits() { + let sem1 = Arc::new(Semaphore::new(3)); + let sem2 = Arc::new(Semaphore::new(3)); + let mut p1 = sem1.try_acquire_owned().unwrap(); + let p2 = sem2.try_acquire_owned().unwrap(); + p1.merge(p2) +} + #[tokio::test] -async fn stresstest() { +#[cfg(feature = "full")] +async fn stress_test() { let sem = Arc::new(Semaphore::new(5)); let mut join_handles = Vec::new(); for _ in 0..1000 { diff --git a/tests/sync_watch.rs b/tests/sync_watch.rs index b7bbaf7..d4f8ce8 100644 --- a/tests/sync_watch.rs +++ b/tests/sync_watch.rs @@ -1,6 +1,9 @@ #![allow(clippy::cognitive_complexity)] #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::watch; use tokio_test::task::spawn; @@ -174,17 +177,24 @@ fn poll_close() { fn borrow_and_update() { let (tx, mut rx) = watch::channel("one"); + assert!(!rx.has_changed().unwrap()); + tx.send("two").unwrap(); + assert!(rx.has_changed().unwrap()); assert_ready!(spawn(rx.changed()).poll()).unwrap(); assert_pending!(spawn(rx.changed()).poll()); + assert!(!rx.has_changed().unwrap()); tx.send("three").unwrap(); + assert!(rx.has_changed().unwrap()); assert_eq!(*rx.borrow_and_update(), "three"); assert_pending!(spawn(rx.changed()).poll()); + assert!(!rx.has_changed().unwrap()); drop(tx); assert_eq!(*rx.borrow_and_update(), "three"); assert_ready!(spawn(rx.changed()).poll()).unwrap_err(); + assert!(rx.has_changed().is_err()); } #[test] @@ -201,3 +211,33 @@ fn reopened_after_subscribe() { drop(rx); assert!(tx.is_closed()); } + +#[test] +#[cfg(panic = "unwind")] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +fn send_modify_panic() { + let (tx, mut rx) = watch::channel("one"); + + tx.send_modify(|old| *old = "two"); + assert_eq!(*rx.borrow_and_update(), "two"); + + let mut rx2 = rx.clone(); + assert_eq!(*rx2.borrow_and_update(), "two"); + + let mut task = spawn(rx2.changed()); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + tx.send_modify(|old| { + *old = "panicked"; + panic!(); + }) + })); + assert!(result.is_err()); + + assert_pending!(task.poll()); + assert_eq!(*rx.borrow(), "panicked"); + + tx.send_modify(|old| *old = "three"); + assert_ready_ok!(task.poll()); + assert_eq!(*rx.borrow_and_update(), "three"); +} diff --git a/tests/task_abort.rs b/tests/task_abort.rs index 06c61dc..8a1b007 100644 --- a/tests/task_abort.rs +++ b/tests/task_abort.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery use std::sync::Arc; use std::thread::sleep; @@ -26,10 +26,7 @@ fn test_abort_without_panic_3157() { .unwrap(); rt.block_on(async move { - let handle = tokio::spawn(async move { - println!("task started"); - tokio::time::sleep(Duration::new(100, 0)).await - }); + let handle = tokio::spawn(async move { tokio::time::sleep(Duration::new(100, 0)).await }); // wait for task to sleep. tokio::time::sleep(Duration::from_millis(10)).await; @@ -89,7 +86,7 @@ fn test_abort_without_panic_3662() { // Note: We do the following to trigger a deferred task cleanup. // // The relevant piece of code you want to look at is in: - // `Inner::block_on` of `basic_scheduler.rs`. + // `Inner::block_on` of `scheduler/current_thread.rs`. // // We cause the cleanup to happen by having a poll return Pending once // so that the scheduler can go into the "auxiliary tasks" mode, at @@ -159,7 +156,6 @@ fn test_abort_wakes_task_3964() { let handle = tokio::spawn(async move { // Make sure the Arc is moved into the task let _notify_dropped = notify_dropped; - println!("task started"); tokio::time::sleep(Duration::new(100, 0)).await }); @@ -188,7 +184,6 @@ fn test_abort_task_that_panics_on_drop_contained() { let handle = tokio::spawn(async move { // Make sure the Arc is moved into the task let _panic_dropped = PanicOnDrop; - println!("task started"); tokio::time::sleep(Duration::new(100, 0)).await }); @@ -213,7 +208,6 @@ fn test_abort_task_that_panics_on_drop_returned() { let handle = tokio::spawn(async move { // Make sure the Arc is moved into the task let _panic_dropped = PanicOnDrop; - println!("task started"); tokio::time::sleep(Duration::new(100, 0)).await }); diff --git a/tests/task_blocking.rs b/tests/task_blocking.rs index e6cde25..d82a0e0 100644 --- a/tests/task_blocking.rs +++ b/tests/task_blocking.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads -use tokio::{runtime, task}; +use tokio::{runtime, task, time}; use tokio_test::assert_ok; use std::thread; @@ -77,7 +77,7 @@ async fn block_in_block() { #[tokio::test(flavor = "current_thread")] #[should_panic] -async fn no_block_in_basic_scheduler() { +async fn no_block_in_current_thread_scheduler() { task::block_in_place(|| {}); } @@ -91,7 +91,7 @@ fn yes_block_in_threaded_block_on() { #[test] #[should_panic] -fn no_block_in_basic_block_on() { +fn no_block_in_current_thread_block_on() { let rt = runtime::Builder::new_current_thread().build().unwrap(); rt.block_on(async { task::block_in_place(|| {}); @@ -99,7 +99,7 @@ fn no_block_in_basic_block_on() { } #[test] -fn can_enter_basic_rt_from_within_block_in_place() { +fn can_enter_current_thread_rt_from_within_block_in_place() { let outer = tokio::runtime::Runtime::new().unwrap(); outer.block_on(async { @@ -227,3 +227,84 @@ fn coop_disabled_in_block_in_place_in_block_on() { done_rx.recv().unwrap().unwrap(); } + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn blocking_when_paused() { + // Do not auto-advance time when we have started a blocking task that has + // not yet finished. + time::timeout( + Duration::from_secs(3), + task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))), + ) + .await + .expect("timeout should not trigger") + .expect("blocking task should finish"); + + // Really: Do not auto-advance time, even if the timeout is short and the + // blocking task runs for longer than that. It doesn't matter: Tokio time + // is paused; system time is not. + time::timeout( + Duration::from_millis(1), + task::spawn_blocking(|| thread::sleep(Duration::from_millis(50))), + ) + .await + .expect("timeout should not trigger") + .expect("blocking task should finish"); +} + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn blocking_task_wakes_paused_runtime() { + let t0 = std::time::Instant::now(); + time::timeout( + Duration::from_secs(15), + task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))), + ) + .await + .expect("timeout should not trigger") + .expect("blocking task should finish"); + assert!( + t0.elapsed() < Duration::from_secs(10), + "completing a spawn_blocking should wake the scheduler if it's parked while time is paused" + ); +} + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn unawaited_blocking_task_wakes_paused_runtime() { + let t0 = std::time::Instant::now(); + + // When this task finishes, time should auto-advance, even though the + // JoinHandle has not been awaited yet. + let a = task::spawn_blocking(|| { + thread::sleep(Duration::from_millis(1)); + }); + + crate::time::sleep(Duration::from_secs(15)).await; + a.await.expect("blocking task should finish"); + assert!( + t0.elapsed() < Duration::from_secs(10), + "completing a spawn_blocking should wake the scheduler if it's parked while time is paused" + ); +} + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn panicking_blocking_task_wakes_paused_runtime() { + let t0 = std::time::Instant::now(); + let result = time::timeout( + Duration::from_secs(15), + task::spawn_blocking(|| { + thread::sleep(Duration::from_millis(1)); + panic!("blocking task panicked"); + }), + ) + .await + .expect("timeout should not trigger"); + assert!(result.is_err(), "blocking task should have panicked"); + assert!( + t0.elapsed() < Duration::from_secs(10), + "completing a spawn_blocking should wake the scheduler if it's parked while time is paused" + ); +} diff --git a/tests/task_builder.rs b/tests/task_builder.rs index 1499abf..78329ff 100644 --- a/tests/task_builder.rs +++ b/tests/task_builder.rs @@ -11,6 +11,7 @@ mod tests { let result = Builder::new() .name("name") .spawn(async { "task executed" }) + .unwrap() .await; assert_eq!(result.unwrap(), "task executed"); @@ -21,6 +22,7 @@ mod tests { let result = Builder::new() .name("name") .spawn_blocking(|| "task executed") + .unwrap() .await; assert_eq!(result.unwrap(), "task executed"); @@ -34,6 +36,7 @@ mod tests { Builder::new() .name("name") .spawn_local(async move { unsend_data }) + .unwrap() .await }) .await; @@ -43,14 +46,20 @@ mod tests { #[test] async fn spawn_without_name() { - let result = Builder::new().spawn(async { "task executed" }).await; + let result = Builder::new() + .spawn(async { "task executed" }) + .unwrap() + .await; assert_eq!(result.unwrap(), "task executed"); } #[test] async fn spawn_blocking_without_name() { - let result = Builder::new().spawn_blocking(|| "task executed").await; + let result = Builder::new() + .spawn_blocking(|| "task executed") + .unwrap() + .await; assert_eq!(result.unwrap(), "task executed"); } @@ -59,7 +68,12 @@ mod tests { async fn spawn_local_without_name() { let unsend_data = Rc::new("task executed"); let result = LocalSet::new() - .run_until(async move { Builder::new().spawn_local(async move { unsend_data }).await }) + .run_until(async move { + Builder::new() + .spawn_local(async move { unsend_data }) + .unwrap() + .await + }) .await; assert_eq!(*result.unwrap(), "task executed"); diff --git a/tests/task_id.rs b/tests/task_id.rs new file mode 100644 index 0000000..d7b7c0c --- /dev/null +++ b/tests/task_id.rs @@ -0,0 +1,303 @@ +#![warn(rust_2018_idioms)] +#![allow(clippy::declare_interior_mutable_const)] +#![cfg(all(feature = "full", tokio_unstable))] + +#[cfg(not(tokio_wasi))] +use std::error::Error; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +#[cfg(not(tokio_wasi))] +use tokio::runtime::{Builder, Runtime}; +use tokio::sync::oneshot; +use tokio::task::{self, Id, LocalSet}; + +#[cfg(not(tokio_wasi))] +mod support { + pub mod panic; +} +#[cfg(not(tokio_wasi))] +use support::panic::test_panic; + +#[tokio::test(flavor = "current_thread")] +async fn task_id_spawn() { + tokio::spawn(async { println!("task id: {}", task::id()) }) + .await + .unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "current_thread")] +async fn task_id_spawn_blocking() { + task::spawn_blocking(|| println!("task id: {}", task::id())) + .await + .unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_collision_current_thread() { + let handle1 = tokio::spawn(async { task::id() }); + let handle2 = tokio::spawn(async { task::id() }); + + let (id1, id2) = tokio::join!(handle1, handle2); + assert_ne!(id1.unwrap(), id2.unwrap()); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_collision_multi_thread() { + let handle1 = tokio::spawn(async { task::id() }); + let handle2 = tokio::spawn(async { task::id() }); + + let (id1, id2) = tokio::join!(handle1, handle2); + assert_ne!(id1.unwrap(), id2.unwrap()); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_ids_match_current_thread() { + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(async { + let id = rx.await.unwrap(); + assert_eq!(id, task::id()); + }); + tx.send(handle.id()).unwrap(); + handle.await.unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_ids_match_multi_thread() { + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(async { + let id = rx.await.unwrap(); + assert_eq!(id, task::id()); + }); + tx.send(handle.id()).unwrap(); + handle.await.unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_future_destructor_completion() { + struct MyFuture { + tx: Option<oneshot::Sender<Id>>, + } + + impl Future for MyFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + Poll::Ready(()) + } + } + + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { tx: Some(tx) }); + let id = handle.id(); + handle.await.unwrap(); + assert_eq!(rx.await.unwrap(), id); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_future_destructor_abort() { + struct MyFuture { + tx: Option<oneshot::Sender<Id>>, + } + + impl Future for MyFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + Poll::Pending + } + } + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { tx: Some(tx) }); + let id = handle.id(); + handle.abort(); + assert!(handle.await.unwrap_err().is_cancelled()); + assert_eq!(rx.await.unwrap(), id); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_output_destructor_handle_dropped_before_completion() { + struct MyOutput { + tx: Option<oneshot::Sender<Id>>, + } + + impl Drop for MyOutput { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + struct MyFuture { + tx: Option<oneshot::Sender<Id>>, + } + + impl Future for MyFuture { + type Output = MyOutput; + + fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> { + Poll::Ready(MyOutput { tx: self.tx.take() }) + } + } + + let (tx, mut rx) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { tx: Some(tx) }); + let id = handle.id(); + drop(handle); + assert!(rx.try_recv().is_err()); + assert_eq!(rx.await.unwrap(), id); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_output_destructor_handle_dropped_after_completion() { + struct MyOutput { + tx: Option<oneshot::Sender<Id>>, + } + + impl Drop for MyOutput { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + struct MyFuture { + tx_output: Option<oneshot::Sender<Id>>, + tx_future: Option<oneshot::Sender<()>>, + } + + impl Future for MyFuture { + type Output = MyOutput; + + fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> { + let _ = self.tx_future.take().unwrap().send(()); + Poll::Ready(MyOutput { + tx: self.tx_output.take(), + }) + } + } + + let (tx_output, mut rx_output) = oneshot::channel(); + let (tx_future, rx_future) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { + tx_output: Some(tx_output), + tx_future: Some(tx_future), + }); + let id = handle.id(); + rx_future.await.unwrap(); + assert!(rx_output.try_recv().is_err()); + drop(handle); + assert_eq!(rx_output.await.unwrap(), id); +} + +#[test] +fn task_try_id_outside_task() { + assert_eq!(None, task::try_id()); +} + +#[cfg(not(tokio_wasi))] +#[test] +fn task_try_id_inside_block_on() { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + assert_eq!(None, task::try_id()); + }); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_spawn_local() { + LocalSet::new() + .run_until(async { + task::spawn_local(async { println!("task id: {}", task::id()) }) + .await + .unwrap(); + }) + .await +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_nested_spawn_local() { + LocalSet::new() + .run_until(async { + task::spawn_local(async { + let parent_id = task::id(); + LocalSet::new() + .run_until(async { + task::spawn_local(async move { + assert_ne!(parent_id, task::id()); + }) + .await + .unwrap(); + }) + .await; + assert_eq!(parent_id, task::id()); + }) + .await + .unwrap(); + }) + .await; +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_block_in_place_block_on_spawn() { + task::spawn(async { + let parent_id = task::id(); + + task::block_in_place(move || { + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(rt.spawn(async move { + assert_ne!(parent_id, task::id()); + })) + .unwrap(); + }); + + assert_eq!(parent_id, task::id()); + }) + .await + .unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[test] +fn task_id_outside_task_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _ = task::id(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[cfg(not(tokio_wasi))] +#[test] +fn task_id_inside_block_on_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + task::id(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/tests/task_join_set.rs b/tests/task_join_set.rs new file mode 100644 index 0000000..b1b6cf9 --- /dev/null +++ b/tests/task_join_set.rs @@ -0,0 +1,235 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full"))] + +use tokio::sync::oneshot; +use tokio::task::JoinSet; +use tokio::time::Duration; + +fn rt() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() +} + +#[tokio::test(start_paused = true)] +async fn test_with_sleep() { + let mut set = JoinSet::new(); + + for i in 0..10 { + set.spawn(async move { i }); + assert_eq!(set.len(), 1 + i); + } + set.detach_all(); + assert_eq!(set.len(), 0); + + assert!(matches!(set.join_next().await, None)); + + for i in 0..10 { + set.spawn(async move { + tokio::time::sleep(Duration::from_secs(i as u64)).await; + i + }); + assert_eq!(set.len(), 1 + i); + } + + let mut seen = [false; 10]; + while let Some(res) = set.join_next().await.transpose().unwrap() { + seen[res] = true; + } + + for was_seen in &seen { + assert!(was_seen); + } + assert!(matches!(set.join_next().await, None)); + + // Do it again. + for i in 0..10 { + set.spawn(async move { + tokio::time::sleep(Duration::from_secs(i as u64)).await; + i + }); + } + + let mut seen = [false; 10]; + while let Some(res) = set.join_next().await.transpose().unwrap() { + seen[res] = true; + } + + for was_seen in &seen { + assert!(was_seen); + } + assert!(matches!(set.join_next().await, None)); +} + +#[tokio::test] +async fn test_abort_on_drop() { + let mut set = JoinSet::new(); + + let mut recvs = Vec::new(); + + for _ in 0..16 { + let (send, recv) = oneshot::channel::<()>(); + recvs.push(recv); + + set.spawn(async { + // This task will never complete on its own. + futures::future::pending::<()>().await; + drop(send); + }); + } + + drop(set); + + for recv in recvs { + // The task is aborted soon and we will receive an error. + assert!(recv.await.is_err()); + } +} + +#[tokio::test] +async fn alternating() { + let mut set = JoinSet::new(); + + assert_eq!(set.len(), 0); + set.spawn(async {}); + assert_eq!(set.len(), 1); + set.spawn(async {}); + assert_eq!(set.len(), 2); + + for _ in 0..16 { + let () = set.join_next().await.unwrap().unwrap(); + assert_eq!(set.len(), 1); + set.spawn(async {}); + assert_eq!(set.len(), 2); + } +} + +#[tokio::test(start_paused = true)] +async fn abort_tasks() { + let mut set = JoinSet::new(); + let mut num_canceled = 0; + let mut num_completed = 0; + for i in 0..16 { + let abort = set.spawn(async move { + tokio::time::sleep(Duration::from_secs(i as u64)).await; + i + }); + + if i % 2 != 0 { + // abort odd-numbered tasks. + abort.abort(); + } + } + loop { + match set.join_next().await { + Some(Ok(res)) => { + num_completed += 1; + assert_eq!(res % 2, 0); + } + Some(Err(e)) => { + assert!(e.is_cancelled()); + num_canceled += 1; + } + None => break, + } + } + + assert_eq!(num_canceled, 8); + assert_eq!(num_completed, 8); +} + +#[test] +fn runtime_gone() { + let mut set = JoinSet::new(); + { + let rt = rt(); + set.spawn_on(async { 1 }, rt.handle()); + drop(rt); + } + + assert!(rt() + .block_on(set.join_next()) + .unwrap() + .unwrap_err() + .is_cancelled()); +} + +#[tokio::test(start_paused = true)] +async fn abort_all() { + let mut set: JoinSet<()> = JoinSet::new(); + + for _ in 0..5 { + set.spawn(futures::future::pending()); + } + for _ in 0..5 { + set.spawn(async { + tokio::time::sleep(Duration::from_secs(1)).await; + }); + } + + // The join set will now have 5 pending tasks and 5 ready tasks. + tokio::time::sleep(Duration::from_secs(2)).await; + + set.abort_all(); + assert_eq!(set.len(), 10); + + let mut count = 0; + while let Some(res) = set.join_next().await { + if let Err(err) = res { + assert!(err.is_cancelled()); + } + count += 1; + } + assert_eq!(count, 10); + assert_eq!(set.len(), 0); +} + +#[cfg(feature = "parking_lot")] +mod parking_lot { + use super::*; + + use futures::future::FutureExt; + + // This ensures that `join_next` works correctly when the coop budget is + // exhausted. + #[tokio::test(flavor = "current_thread")] + async fn join_set_coop() { + // Large enough to trigger coop. + const TASK_NUM: u32 = 1000; + + static SEM: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(0); + + let mut set = JoinSet::new(); + + for _ in 0..TASK_NUM { + set.spawn(async { + SEM.add_permits(1); + }); + } + + // Wait for all tasks to complete. + // + // Since this is a `current_thread` runtime, there's no race condition + // between the last permit being added and the task completing. + let _ = SEM.acquire_many(TASK_NUM).await.unwrap(); + + let mut count = 0; + let mut coop_count = 0; + loop { + match set.join_next().now_or_never() { + Some(Some(Ok(()))) => {} + Some(Some(Err(err))) => panic!("failed: {}", err), + None => { + coop_count += 1; + tokio::task::yield_now().await; + continue; + } + Some(None) => break, + } + + count += 1; + } + assert!(coop_count >= 1); + assert_eq!(count, TASK_NUM); + } +} diff --git a/tests/task_local.rs b/tests/task_local.rs index 9981532..949a40c 100644 --- a/tests/task_local.rs +++ b/tests/task_local.rs @@ -1,10 +1,17 @@ -tokio::task_local! { - static REQ_ID: u32; - pub static FOO: bool; -} +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads +#![allow(clippy::declare_interior_mutable_const)] +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::sync::oneshot; #[tokio::test(flavor = "multi_thread")] async fn local() { + tokio::task_local! { + static REQ_ID: u32; + pub static FOO: bool; + } + let j1 = tokio::spawn(REQ_ID.scope(1, async move { assert_eq!(REQ_ID.get(), 1); assert_eq!(REQ_ID.get(), 1); @@ -29,3 +36,84 @@ async fn local() { j2.await.unwrap(); j3.await.unwrap(); } + +#[tokio::test] +async fn task_local_available_on_abort() { + tokio::task_local! { + static KEY: u32; + } + + struct MyFuture { + tx_poll: Option<oneshot::Sender<()>>, + tx_drop: Option<oneshot::Sender<u32>>, + } + impl Future for MyFuture { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + if let Some(tx_poll) = self.tx_poll.take() { + let _ = tx_poll.send(()); + } + Poll::Pending + } + } + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx_drop.take().unwrap().send(KEY.get()); + } + } + + let (tx_drop, rx_drop) = oneshot::channel(); + let (tx_poll, rx_poll) = oneshot::channel(); + + let h = tokio::spawn(KEY.scope( + 42, + MyFuture { + tx_poll: Some(tx_poll), + tx_drop: Some(tx_drop), + }, + )); + + rx_poll.await.unwrap(); + h.abort(); + assert_eq!(rx_drop.await.unwrap(), 42); + + let err = h.await.unwrap_err(); + if !err.is_cancelled() { + if let Ok(panic) = err.try_into_panic() { + std::panic::resume_unwind(panic); + } else { + panic!(); + } + } +} + +#[tokio::test] +async fn task_local_available_on_completion_drop() { + tokio::task_local! { + static KEY: u32; + } + + struct MyFuture { + tx: Option<oneshot::Sender<u32>>, + } + impl Future for MyFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + Poll::Ready(()) + } + } + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(KEY.get()); + } + } + + let (tx, rx) = oneshot::channel(); + + let h = tokio::spawn(KEY.scope(42, MyFuture { tx: Some(tx) })); + + assert_eq!(rx.await.unwrap(), 42); + h.await.unwrap(); +} diff --git a/tests/task_local_set.rs b/tests/task_local_set.rs index f8a35d0..2da87f5 100644 --- a/tests/task_local_set.rs +++ b/tests/task_local_set.rs @@ -6,18 +6,23 @@ use futures::{ FutureExt, }; -use tokio::runtime::{self, Runtime}; +use tokio::runtime; use tokio::sync::{mpsc, oneshot}; use tokio::task::{self, LocalSet}; use tokio::time; +#[cfg(not(tokio_wasi))] use std::cell::Cell; -use std::sync::atomic::Ordering::{self, SeqCst}; -use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::atomic::AtomicBool; +#[cfg(not(tokio_wasi))] +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +#[cfg(not(tokio_wasi))] +use std::sync::atomic::Ordering::SeqCst; use std::time::Duration; #[tokio::test(flavor = "current_thread")] -async fn local_basic_scheduler() { +async fn local_current_thread_scheduler() { LocalSet::new() .run_until(async { task::spawn_local(async {}).await.unwrap(); @@ -25,6 +30,7 @@ async fn local_basic_scheduler() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn local_threadpool() { thread_local! { @@ -45,6 +51,7 @@ async fn local_threadpool() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn localset_future_threadpool() { thread_local! { @@ -60,6 +67,7 @@ async fn localset_future_threadpool() { local.await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn localset_future_timers() { static RAN1: AtomicBool = AtomicBool::new(false); @@ -104,6 +112,7 @@ async fn localset_future_drives_all_local_futs() { assert!(RAN3.load(Ordering::SeqCst)); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn local_threadpool_timer() { // This test ensures that runtime services like the timer are properly @@ -126,7 +135,23 @@ async fn local_threadpool_timer() { }) .await; } +#[test] +fn enter_guard_spawn() { + let local = LocalSet::new(); + let _guard = local.enter(); + // Run the local task set. + let join = task::spawn_local(async { true }); + let rt = runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + local.block_on(&rt, async move { + assert!(join.await.unwrap()); + }); +} + +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[test] // This will panic, since the thread that calls `block_on` cannot use // in-place blocking inside of `block_on`. @@ -153,6 +178,7 @@ fn local_threadpool_blocking_in_place() { }); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn local_threadpool_blocking_run() { thread_local! { @@ -181,6 +207,7 @@ async fn local_threadpool_blocking_run() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn all_spawns_are_local() { use futures::future; @@ -207,6 +234,7 @@ async fn all_spawns_are_local() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn nested_spawn_is_local() { thread_local! { @@ -242,6 +270,7 @@ async fn nested_spawn_is_local() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[test] fn join_local_future_elsewhere() { thread_local! { @@ -255,14 +284,12 @@ fn join_local_future_elsewhere() { local.block_on(&rt, async move { let (tx, rx) = oneshot::channel(); let join = task::spawn_local(async move { - println!("hello world running..."); assert!( ON_RT_THREAD.with(|cell| cell.get()), "local task must run on local thread, no matter where it is awaited" ); rx.await.unwrap(); - println!("hello world task done"); "hello world" }); let join2 = task::spawn(async move { @@ -272,16 +299,34 @@ fn join_local_future_elsewhere() { ); tx.send(()).expect("task shouldn't have ended yet"); - println!("waking up hello world..."); join.await.expect("task should complete successfully"); - - println!("hello world task joined"); }); join2.await.unwrap() }); } +// Tests for <https://github.com/tokio-rs/tokio/issues/4973> +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads +#[tokio::test(flavor = "multi_thread")] +async fn localset_in_thread_local() { + thread_local! { + static LOCAL_SET: LocalSet = LocalSet::new(); + } + + // holds runtime thread until end of main fn. + let (_tx, rx) = oneshot::channel::<()>(); + let handle = tokio::runtime::Handle::current(); + + std::thread::spawn(move || { + LOCAL_SET.with(|local_set| { + handle.block_on(local_set.run_until(async move { + let _ = rx.await; + })) + }); + }); +} + #[test] fn drop_cancels_tasks() { use std::rc::Rc; @@ -345,9 +390,7 @@ fn with_timeout(timeout: Duration, f: impl FnOnce() + Send + 'static) { ), // Did the test thread panic? We'll find out for sure when we `join` // with it. - Err(RecvTimeoutError::Disconnected) => { - println!("done_rx dropped, did the test thread panic?"); - } + Err(RecvTimeoutError::Disconnected) => {} // Test completed successfully! Ok(()) => {} } @@ -355,6 +398,7 @@ fn with_timeout(timeout: Duration, f: impl FnOnce() + Send + 'static) { thread.join().expect("test thread should not panic!") } +#[cfg_attr(tokio_wasi, ignore = "`unwrap()` in `with_timeout()` panics on Wasi")] #[test] fn drop_cancels_remote_tasks() { // This test reproduces issue #1885. @@ -377,6 +421,10 @@ fn drop_cancels_remote_tasks() { }); } +#[cfg_attr( + tokio_wasi, + ignore = "FIXME: `task::spawn_local().await.unwrap()` panics on Wasi" +)] #[test] fn local_tasks_wake_join_all() { // This test reproduces issue #2460. @@ -398,6 +446,7 @@ fn local_tasks_wake_join_all() { }); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[test] fn local_tasks_are_polled_after_tick() { // This test depends on timing, so we run it up to five times. @@ -414,6 +463,7 @@ fn local_tasks_are_polled_after_tick() { local_tasks_are_polled_after_tick_inner(); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[tokio::main(flavor = "current_thread")] async fn local_tasks_are_polled_after_tick_inner() { // Reproduces issues #1899 and #1900 @@ -450,12 +500,15 @@ async fn local_tasks_are_polled_after_tick_inner() { tx.send(()).unwrap(); } - time::sleep(Duration::from_millis(20)).await; - let rx1 = RX1.load(SeqCst); - let rx2 = RX2.load(SeqCst); - println!("EXPECT = {}; RX1 = {}; RX2 = {}", EXPECTED, rx1, rx2); - assert_eq!(EXPECTED, rx1); - assert_eq!(EXPECTED, rx2); + loop { + time::sleep(Duration::from_millis(20)).await; + let rx1 = RX1.load(SeqCst); + let rx2 = RX2.load(SeqCst); + + if rx1 == EXPECTED && rx2 == EXPECTED { + break; + } + } }); while let Some(oneshot) = rx.recv().await { @@ -517,7 +570,122 @@ async fn spawn_wakes_localset() { } } -fn rt() -> Runtime { +#[test] +fn store_local_set_in_thread_local_with_runtime() { + use tokio::runtime::Runtime; + + thread_local! { + static CURRENT: RtAndLocalSet = RtAndLocalSet::new(); + } + + struct RtAndLocalSet { + rt: Runtime, + local: LocalSet, + } + + impl RtAndLocalSet { + fn new() -> RtAndLocalSet { + RtAndLocalSet { + rt: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(), + local: LocalSet::new(), + } + } + + async fn inner_method(&self) { + self.local + .run_until(async move { + tokio::task::spawn_local(async {}); + }) + .await + } + + fn method(&self) { + self.rt.block_on(self.inner_method()); + } + } + + CURRENT.with(|f| { + f.method(); + }); +} + +#[cfg(tokio_unstable)] +mod unstable { + use tokio::runtime::UnhandledPanic; + use tokio::task::LocalSet; + + #[tokio::test] + #[should_panic( + expected = "a spawned task panicked and the LocalSet is configured to shutdown on unhandled panic" + )] + async fn shutdown_on_panic() { + LocalSet::new() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .run_until(async { + tokio::task::spawn_local(async { + panic!("boom"); + }); + + futures::future::pending::<()>().await; + }) + .await; + } + + // This test compares that, when the task driving `run_until` has already + // consumed budget, the `run_until` future has less budget than a "spawned" + // task. + // + // "Budget" is a fuzzy metric as the Tokio runtime is able to change values + // internally. This is why the test uses indirection to test this. + #[tokio::test] + async fn run_until_does_not_get_own_budget() { + // Consume some budget + tokio::task::consume_budget().await; + + LocalSet::new() + .run_until(async { + let spawned = tokio::spawn(async { + let mut spawned_n = 0; + + { + let mut spawned = tokio_test::task::spawn(async { + loop { + spawned_n += 1; + tokio::task::consume_budget().await; + } + }); + // Poll once + assert!(!spawned.poll().is_ready()); + } + + spawned_n + }); + + let mut run_until_n = 0; + { + let mut run_until = tokio_test::task::spawn(async { + loop { + run_until_n += 1; + tokio::task::consume_budget().await; + } + }); + // Poll once + assert!(!run_until.poll().is_ready()); + } + + let spawned_n = spawned.await.unwrap(); + assert_ne!(spawned_n, 0); + assert_ne!(run_until_n, 0); + assert!(spawned_n > run_until_n); + }) + .await + } +} + +fn rt() -> runtime::Runtime { tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/tests/task_panic.rs b/tests/task_panic.rs new file mode 100644 index 0000000..e4cedce --- /dev/null +++ b/tests/task_panic.rs @@ -0,0 +1,123 @@ +#![warn(rust_2018_idioms)] +#![allow(clippy::declare_interior_mutable_const)] +#![cfg(all(feature = "full", not(tokio_wasi)))] + +use futures::future; +use std::error::Error; +use tokio::runtime::Builder; +use tokio::task::{self, block_in_place}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn block_in_place_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = Builder::new_current_thread().enable_all().build().unwrap(); + rt.block_on(async { + block_in_place(|| {}); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_set_spawn_local_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _local = task::LocalSet::new(); + + let _ = task::spawn_local(async {}); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_set_block_on_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = Builder::new_current_thread().enable_all().build().unwrap(); + let local = task::LocalSet::new(); + + rt.block_on(async { + local.block_on(&rt, future::pending::<()>()); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn spawn_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + tokio::spawn(future::pending::<()>()); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_key_sync_scope_panic_caller() -> Result<(), Box<dyn Error>> { + tokio::task_local! { + static NUMBER: u32; + } + + let panic_location_file = test_panic(|| { + NUMBER.sync_scope(1, || { + NUMBER.with(|_| { + NUMBER.sync_scope(1, || {}); + }); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_key_with_panic_caller() -> Result<(), Box<dyn Error>> { + tokio::task_local! { + static NUMBER: u32; + } + + let panic_location_file = test_panic(|| { + NUMBER.with(|_| {}); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_key_get_panic_caller() -> Result<(), Box<dyn Error>> { + tokio::task_local! { + static NUMBER: u32; + } + + let panic_location_file = test_panic(|| { + NUMBER.get(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/tests/tcp_accept.rs b/tests/tcp_accept.rs index 5ffb946..ba4a498 100644 --- a/tests/tcp_accept.rs +++ b/tests/tcp_accept.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{mpsc, oneshot}; diff --git a/tests/tcp_connect.rs b/tests/tcp_connect.rs index cbe68fa..f238442 100644 --- a/tests/tcp_connect.rs +++ b/tests/tcp_connect.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::net::{TcpListener, TcpStream}; use tokio::sync::oneshot; @@ -35,6 +35,7 @@ async fn connect_v4() { } #[tokio::test] +#[cfg(not(tokio_no_ipv6))] async fn connect_v6() { let srv = assert_ok!(TcpListener::bind("[::1]:0").await); let addr = assert_ok!(srv.local_addr()); diff --git a/tests/tcp_echo.rs b/tests/tcp_echo.rs index 5bb7ff0..f47d4ac 100644 --- a/tests/tcp_echo.rs +++ b/tests/tcp_echo.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::{self, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; diff --git a/tests/tcp_into_split.rs b/tests/tcp_into_split.rs index 2e06643..010984a 100644 --- a/tests/tcp_into_split.rs +++ b/tests/tcp_into_split.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use std::io::{Error, ErrorKind, Result}; use std::io::{Read, Write}; diff --git a/tests/tcp_into_std.rs b/tests/tcp_into_std.rs index 4bf24c1..320d994 100644 --- a/tests/tcp_into_std.rs +++ b/tests/tcp_into_std.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use std::io::Read; use std::io::Result; diff --git a/tests/tcp_peek.rs b/tests/tcp_peek.rs index aecc0ac..b712023 100644 --- a/tests/tcp_peek.rs +++ b/tests/tcp_peek.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::AsyncReadExt; use tokio::net::TcpStream; @@ -15,7 +15,7 @@ async fn peek() { let addr = listener.local_addr().unwrap(); let t = thread::spawn(move || assert_ok!(listener.accept()).0); - let left = net::TcpStream::connect(&addr).unwrap(); + let left = net::TcpStream::connect(addr).unwrap(); let mut right = t.join().unwrap(); let _ = right.write(&[1, 2, 3, 4]).unwrap(); diff --git a/tests/tcp_shutdown.rs b/tests/tcp_shutdown.rs index 536a161..1d477f3 100644 --- a/tests/tcp_shutdown.rs +++ b/tests/tcp_shutdown.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::{self, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; diff --git a/tests/tcp_socket.rs b/tests/tcp_socket.rs index 9258864..66309d3 100644 --- a/tests/tcp_socket.rs +++ b/tests/tcp_socket.rs @@ -1,6 +1,7 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind +use std::time::Duration; use tokio::net::TcpSocket; use tokio_test::assert_ok; @@ -23,6 +24,7 @@ async fn basic_usage_v4() { } #[tokio::test] +#[cfg(not(tokio_no_ipv6))] async fn basic_usage_v6() { // Create server let addr = assert_ok!("[::1]:0".parse()); @@ -58,3 +60,16 @@ async fn bind_before_connect() { // Accept let _ = assert_ok!(srv.accept().await); } + +#[tokio::test] +async fn basic_linger() { + // Create server + let addr = assert_ok!("127.0.0.1:0".parse()); + let srv = assert_ok!(TcpSocket::new_v4()); + assert_ok!(srv.bind(addr)); + + assert!(srv.linger().unwrap().is_none()); + + srv.set_linger(Some(Duration::new(0, 0))).unwrap(); + assert_eq!(srv.linger().unwrap(), Some(Duration::new(0, 0))); +} diff --git a/tests/tcp_split.rs b/tests/tcp_split.rs index 7171dac..335b21f 100644 --- a/tests/tcp_split.rs +++ b/tests/tcp_split.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use std::io::Result; use std::io::{Read, Write}; @@ -36,7 +36,7 @@ async fn split() -> Result<()> { assert_eq!(peek_len1, read_len); assert_eq!(&read_buf[..read_len], MSG); - write_half.write(MSG).await?; + assert_eq!(write_half.write(MSG).await?, MSG.len()); handle.join().unwrap(); Ok(()) } diff --git a/tests/tcp_stream.rs b/tests/tcp_stream.rs index 0b5d12a..31fe3ba 100644 --- a/tests/tcp_stream.rs +++ b/tests/tcp_stream.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::{AsyncReadExt, AsyncWriteExt, Interest}; use tokio::net::{TcpListener, TcpStream}; @@ -254,30 +254,34 @@ async fn create_pair() -> (TcpStream, TcpStream) { (client, server) } -fn read_until_pending(stream: &mut TcpStream) { +fn read_until_pending(stream: &mut TcpStream) -> usize { let mut buf = vec![0u8; 1024 * 1024]; + let mut total = 0; loop { match stream.try_read(&mut buf) { - Ok(_) => (), + Ok(n) => total += n, Err(err) => { assert_eq!(err.kind(), io::ErrorKind::WouldBlock); break; } } } + total } -fn write_until_pending(stream: &mut TcpStream) { +fn write_until_pending(stream: &mut TcpStream) -> usize { let buf = vec![0u8; 1024 * 1024]; + let mut total = 0; loop { match stream.try_write(&buf) { - Ok(_) => (), + Ok(n) => total += n, Err(err) => { assert_eq!(err.kind(), io::ErrorKind::WouldBlock); break; } } } + total } #[tokio::test] @@ -357,3 +361,40 @@ async fn try_read_buf() { } } } + +// read_closed is a best effort event, so test only for no false positives. +#[tokio::test] +async fn read_closed() { + let (client, mut server) = create_pair().await; + + let mut ready_fut = task::spawn(client.ready(Interest::READABLE)); + assert_pending!(ready_fut.poll()); + + assert_ok!(server.write_all(b"ping").await); + + let ready_event = assert_ok!(ready_fut.await); + + assert!(!ready_event.is_read_closed()); +} + +// write_closed is a best effort event, so test only for no false positives. +#[tokio::test] +async fn write_closed() { + let (mut client, mut server) = create_pair().await; + + // Fill the write buffer. + let write_size = write_until_pending(&mut client); + let mut ready_fut = task::spawn(client.ready(Interest::WRITABLE)); + assert_pending!(ready_fut.poll()); + + // Drain the socket to make client writable. + let mut read_size = 0; + while read_size < write_size { + server.readable().await.unwrap(); + read_size += read_until_pending(&mut server); + } + + let ready_event = assert_ok!(ready_fut.await); + + assert!(!ready_event.is_write_closed()); +} diff --git a/tests/time_interval.rs b/tests/time_interval.rs index 5f7bf55..186582e 100644 --- a/tests/time_interval.rs +++ b/tests/time_interval.rs @@ -166,6 +166,42 @@ async fn skip() { check_interval_poll!(i, start, 1800); } +#[tokio::test(start_paused = true)] +async fn reset() { + let start = Instant::now(); + + // This is necessary because the timer is only so granular, and in order for + // all our ticks to resolve, the time needs to be 1ms ahead of what we + // expect, so that the runtime will see that it is time to resolve the timer + time::advance(ms(1)).await; + + let mut i = task::spawn(time::interval_at(start, ms(300))); + + check_interval_poll!(i, start, 0); + + time::advance(ms(100)).await; + check_interval_poll!(i, start); + + time::advance(ms(200)).await; + check_interval_poll!(i, start, 300); + + time::advance(ms(100)).await; + check_interval_poll!(i, start); + + i.reset(); + + time::advance(ms(250)).await; + check_interval_poll!(i, start); + + time::advance(ms(50)).await; + // We add one because when using `reset` method, `Interval` adds the + // `period` from `Instant::now()`, which will always be off by one + check_interval_poll!(i, start, 701); + + time::advance(ms(300)).await; + check_interval_poll!(i, start, 1001); +} + fn poll_next(interval: &mut task::Spawn<time::Interval>) -> Poll<Instant> { interval.enter(|cx, mut interval| interval.poll_tick(cx)) } diff --git a/tests/time_panic.rs b/tests/time_panic.rs new file mode 100644 index 0000000..aaff11b --- /dev/null +++ b/tests/time_panic.rs @@ -0,0 +1,93 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery + +use futures::future; +use std::error::Error; +use std::time::Duration; +use tokio::runtime::{Builder, Runtime}; +use tokio::time::{self, interval, interval_at, timeout, Instant}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn pause_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + + rt.block_on(async { + time::pause(); + time::pause(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn resume_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + + rt.block_on(async { + time::resume(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn interval_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _ = interval(Duration::from_millis(0)); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn interval_at_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + let _ = interval_at(Instant::now(), Duration::from_millis(0)); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn timeout_panic_caller() -> Result<(), Box<dyn Error>> { + let panic_location_file = test_panic(|| { + // Runtime without `enable_time` so it has no current timer set. + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(async { + let _ = timeout(Duration::from_millis(5), future::pending::<()>()); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +fn current_thread() -> Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} diff --git a/tests/time_pause.rs b/tests/time_pause.rs index 02e050a..c772e38 100644 --- a/tests/time_pause.rs +++ b/tests/time_pause.rs @@ -4,7 +4,10 @@ use rand::SeedableRng; use rand::{rngs::StdRng, Rng}; use tokio::time::{self, Duration, Instant, Sleep}; -use tokio_test::{assert_elapsed, assert_err, assert_pending, assert_ready, assert_ready_eq, task}; +use tokio_test::{assert_elapsed, assert_pending, assert_ready, assert_ready_eq, task}; + +#[cfg(not(tokio_wasi))] +use tokio_test::assert_err; use std::{ future::Future, @@ -26,12 +29,14 @@ async fn pause_time_in_task() { t.await.unwrap(); } +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[should_panic] async fn pause_time_in_main_threads() { tokio::time::pause(); } +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn pause_time_in_spawn_threads() { let t = tokio::spawn(async { diff --git a/tests/time_rt.rs b/tests/time_rt.rs index 23367be..20f9e18 100644 --- a/tests/time_rt.rs +++ b/tests/time_rt.rs @@ -5,6 +5,7 @@ use tokio::time::*; use std::sync::mpsc; +#[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] // Wasi doesn't support threads #[test] fn timer_with_threaded_runtime() { use tokio::runtime::Runtime; @@ -25,7 +26,7 @@ fn timer_with_threaded_runtime() { } #[test] -fn timer_with_basic_scheduler() { +fn timer_with_current_thread_scheduler() { use tokio::runtime::Builder; let rt = Builder::new_current_thread().enable_all().build().unwrap(); diff --git a/tests/time_sleep.rs b/tests/time_sleep.rs index 20477d2..4174a73 100644 --- a/tests/time_sleep.rs +++ b/tests/time_sleep.rs @@ -168,6 +168,7 @@ async fn reset_sleep_to_past() { assert_ready!(sleep.poll()); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[test] #[should_panic] fn creating_sleep_outside_of_context() { @@ -188,10 +189,7 @@ async fn greater_than_max() { #[tokio::test] async fn short_sleeps() { - for i in 0..10000 { - if (i % 10) == 0 { - eprintln!("=== {}", i); - } + for _ in 0..10000 { tokio::time::sleep(std::time::Duration::from_millis(0)).await; } } @@ -236,22 +234,6 @@ async fn long_sleeps() { } #[tokio::test] -#[should_panic(expected = "Duration too far into the future")] -async fn very_long_sleeps() { - tokio::time::pause(); - - // Some platforms (eg macos) can't represent times this far in the future - if let Some(deadline) = tokio::time::Instant::now().checked_add(Duration::from_secs(1u64 << 62)) - { - tokio::time::sleep_until(deadline).await; - } else { - // make it pass anyway (we can't skip/ignore the test based on the - // result of checked_add) - panic!("Duration too far into the future (test ignored)") - } -} - -#[tokio::test] async fn reset_after_firing() { let timer = tokio::time::sleep(std::time::Duration::from_millis(1)); tokio::pin!(timer); @@ -333,18 +315,18 @@ async fn drop_from_wake() { tokio::time::pause(); - let mut lock = list.lock().unwrap(); + { + let mut lock = list.lock().unwrap(); - for _ in 0..100 { - let mut timer = Box::pin(tokio::time::sleep(Duration::from_millis(10))); + for _ in 0..100 { + let mut timer = Box::pin(tokio::time::sleep(Duration::from_millis(10))); - let _ = timer.as_mut().poll(&mut Context::from_waker(&arc_wake)); + let _ = timer.as_mut().poll(&mut Context::from_waker(&arc_wake)); - lock.push(timer); + lock.push(timer); + } } - drop(lock); - tokio::time::sleep(Duration::from_millis(11)).await; assert!( diff --git a/tests/time_timeout.rs b/tests/time_timeout.rs index dbd80eb..be6b8fb 100644 --- a/tests/time_timeout.rs +++ b/tests/time_timeout.rs @@ -17,6 +17,7 @@ async fn simultaneous_deadline_future_completion() { assert_ready_ok!(fut.poll()); } +#[cfg_attr(tokio_wasi, ignore = "FIXME: `fut.poll()` panics on Wasi")] #[tokio::test] async fn completed_future_past_deadline() { // Wrap it with a deadline @@ -135,3 +136,16 @@ async fn deadline_future_elapses() { fn ms(n: u64) -> Duration { Duration::from_millis(n) } + +#[tokio::test] +async fn timeout_is_not_exhausted_by_future() { + let fut = timeout(ms(1), async { + let mut buffer = [0u8; 1]; + loop { + use tokio::io::AsyncReadExt; + let _ = tokio::io::empty().read(&mut buffer).await; + } + }); + + assert!(fut.await.is_err()); +} diff --git a/tests/udp.rs b/tests/udp.rs index ec2a1e9..2b6ab4d 100644 --- a/tests/udp.rs +++ b/tests/udp.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind or UDP use futures::future::poll_fn; use std::io; @@ -39,7 +39,7 @@ async fn send_recv_poll() -> std::io::Result<()> { let mut recv_buf = [0u8; 32]; let mut read = ReadBuf::new(&mut recv_buf); - let _len = poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; + poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; assert_eq!(read.filled(), MSG); Ok(()) diff --git a/tests/uds_datagram.rs b/tests/uds_datagram.rs index 5e5486b..c08bd45 100644 --- a/tests/uds_datagram.rs +++ b/tests/uds_datagram.rs @@ -29,9 +29,7 @@ async fn echo() -> io::Result<()> { let server_socket = UnixDatagram::bind(server_path.clone())?; tokio::spawn(async move { - if let Err(e) = echo_server(server_socket).await { - eprintln!("Error in echo server: {}", e); - } + let _ = echo_server(server_socket).await; }); { @@ -55,9 +53,7 @@ async fn echo_from() -> io::Result<()> { let server_socket = UnixDatagram::bind(server_path.clone())?; tokio::spawn(async move { - if let Err(e) = echo_server(server_socket).await { - eprintln!("Error in echo server: {}", e); - } + let _ = echo_server(server_socket).await; }); { @@ -181,7 +177,7 @@ async fn send_recv_poll() -> std::io::Result<()> { let mut recv_buf = [0u8; 32]; let mut read = ReadBuf::new(&mut recv_buf); - let _len = poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; + poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; assert_eq!(read.filled(), msg); Ok(()) diff --git a/tests/uds_stream.rs b/tests/uds_stream.rs index 5f1b4cf..b8c4e6a 100644 --- a/tests/uds_stream.rs +++ b/tests/uds_stream.rs @@ -25,13 +25,13 @@ async fn accept_read_write() -> std::io::Result<()> { let connect = UnixStream::connect(&sock_path); let ((mut server, _), mut client) = try_join(accept, connect).await?; - // Write to the client. TODO: Switch to write_all. - let write_len = client.write(b"hello").await?; - assert_eq!(write_len, 5); + // Write to the client. + client.write_all(b"hello").await?; drop(client); - // Read from the server. TODO: Switch to read_to_end. - let mut buf = [0u8; 5]; - server.read_exact(&mut buf).await?; + + // Read from the server. + let mut buf = vec![]; + server.read_to_end(&mut buf).await?; assert_eq!(&buf, b"hello"); let len = server.read(&mut buf).await?; assert_eq!(len, 0); diff --git a/tests/unwindsafe.rs b/tests/unwindsafe.rs new file mode 100644 index 0000000..3e63820 --- /dev/null +++ b/tests/unwindsafe.rs @@ -0,0 +1,42 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery + +use std::panic::{RefUnwindSafe, UnwindSafe}; + +#[test] +fn notify_is_unwind_safe() { + is_unwind_safe::<tokio::sync::Notify>(); +} + +#[test] +fn join_handle_is_unwind_safe() { + is_unwind_safe::<tokio::task::JoinHandle<()>>(); +} + +#[test] +fn net_types_are_unwind_safe() { + is_unwind_safe::<tokio::net::TcpListener>(); + is_unwind_safe::<tokio::net::TcpSocket>(); + is_unwind_safe::<tokio::net::TcpStream>(); + is_unwind_safe::<tokio::net::UdpSocket>(); +} + +#[test] +#[cfg(unix)] +fn unix_net_types_are_unwind_safe() { + is_unwind_safe::<tokio::net::UnixDatagram>(); + is_unwind_safe::<tokio::net::UnixListener>(); + is_unwind_safe::<tokio::net::UnixStream>(); +} + +#[test] +#[cfg(windows)] +fn windows_net_types_are_unwind_safe() { + use tokio::net::windows::named_pipe::NamedPipeClient; + use tokio::net::windows::named_pipe::NamedPipeServer; + + is_unwind_safe::<NamedPipeClient>(); + is_unwind_safe::<NamedPipeServer>(); +} + +fn is_unwind_safe<T: UnwindSafe + RefUnwindSafe>() {} |