Skip to content

Redundant DNS lookups #81

@alecmocatta

Description

@alecmocatta

Hyper's DNS resolver is spawning a bunch of blocking tasks to look up the same host, causing hundreds of blocking threads to be spawned. ~100 threads with the following stack cause cloudfront_logs to take ~20s to exit after returning from async fn main.

    2427 Thread_7871772: tokio-runtime-worker
    + 2427 thread_start  (in libsystem_pthread.dylib) + 13  [0x7fff63c2840d]
    +   2427 _pthread_start  (in libsystem_pthread.dylib) + 66  [0x7fff63c2c249]
    +     2427 _pthread_body  (in libsystem_pthread.dylib) + 126  [0x7fff63c292eb]
    +       2427 std::sys::unix::thread::Thread::new::thread_start::h545d31fdf79b3d9e  (in cloudfront_logs) + 45  [0x10d1eb3dd]  boxed.rs:1081
    +         2427 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h24d0ac9ca2e34eab  (in cloudfront_logs) + 17  [0x10d053ba1]  function.rs:233
    +           2427 std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::h606d44b084314330  (in cloudfront_logs) + 278  [0x10d020416]  mod.rs:474
    +             2427 std::panic::catch_unwind::hf778b52e64c7dd68  (in cloudfront_logs) + 49  [0x10d01f391]  panic.rs:394
    +               2427 std::panicking::try::h0b8958571d7c7f99  (in cloudfront_logs) + 206  [0x10d05fb8e]  panicking.rs:325
    +                 2427 __rust_try  (in cloudfront_logs) + 29  [0x10d06454d]
    +                   2427 std::panicking::try::do_call::h417c11b07e629a5e  (in cloudfront_logs) + 95  [0x10d05fe8f]  panicking.rs:348
    +                     2427 _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h6187c27ab70b5642  (in cloudfront_logs) + 49  [0x10d01efa1]  panic.rs:318
    +                       2427 std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::ha5c6d4cb1a10bf6f  (in cloudfront_logs) + 49  [0x10d0205c1]  mod.rs:475
    +                         2427 std::sys_common::backtrace::__rust_begin_short_backtrace::h3308c5c2d8decb9c  (in cloudfront_logs) + 49  [0x10cff0371]  backtrace.rs:130
    +                           2427 tokio::runtime::blocking::pool::Spawner::spawn_thread::_$u7b$$u7b$closure$u7d$$u7d$::hb8d26712d1e0a87a  (in cloudfront_logs) + 41  [0x10d0865e9]  pool.rs:209
    +                             2427 tokio::runtime::handle::Handle::enter::h925b673bbbf60922  (in cloudfront_logs) + 82  [0x10d069852]  handle.rs:76
    +                               2427 tokio::runtime::context::enter::h594ce19fae3645d3  (in cloudfront_logs) + 86  [0x10d0144f6]  context.rs:72
    +                                 2427 tokio::runtime::blocking::pool::Spawner::spawn_thread::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h15b91bd6f6a228cc  (in cloudfront_logs) + 55  [0x10d086567]  pool.rs:210
    +                                   2427 tokio::runtime::blocking::pool::Inner::run::h2223211aa8f5c30d  (in cloudfront_logs) + 390  [0x10d0867a6]  pool.rs:230
    +                                     2427 tokio::runtime::task::Notified$LT$S$GT$::run::h7ee3dc3c18e82cba  (in cloudfront_logs) + 29  [0x10d062c0d]  mod.rs:169
    +                                       2427 tokio::runtime::task::raw::RawTask::poll::h26d954b18474d348  (in cloudfront_logs) + 46  [0x10d06b61e]  raw.rs:66
    +                                         2427 tokio::runtime::task::raw::poll::h0fa0afc3e981bb4b  (in cloudfront_logs) + 34  [0x10ceae572]  raw.rs:104
    +                                           2427 tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::hf9c36b55740bbfed  (in cloudfront_logs) + 380  [0x10ced9cbc]  harness.rs:89
    +                                             2427 std::panic::catch_unwind::hbca52d9256b52aa9  (in cloudfront_logs) + 28  [0x10cec72fc]  panic.rs:394
    +                                               2427 std::panicking::try::hd71536d6d35fb12e  (in cloudfront_logs) + 135  [0x10cee18d7]  panicking.rs:325
    +                                                 2427 __rust_try  (in cloudfront_logs) + 29  [0x10cee8ebd]
    +                                                   2427 std::panicking::try::do_call::h46c34f15157635e7  (in cloudfront_logs) + 89  [0x10cee1ac9]  panicking.rs:348
    +                                                     2427 _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h55e08800d1bb0a52  (in cloudfront_logs) + 28  [0x10cec6d0c]  panic.rs:318
    +                                                       2427 core::ops::function::FnOnce::call_once::hf27147462984b278  (in cloudfront_logs) + 32  [0x10cee2290]  function.rs:233
    +                                                         2427 tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::_$u7b$$u7b$closure$u7d$$u7d$::h81c651632a29d4ef  (in cloudfront_logs) + 306  [0x10ceda102]  harness.rs:107
    +                                                           2427 tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::h7b7d24a1533992f2  (in cloudfront_logs) + 54  [0x10cebac16]  core.rs:158
    +                                                             2427 tokio::loom::std::unsafe_cell::UnsafeCell$LT$T$GT$::with_mut::h377f838352fbf4a0  (in cloudfront_logs) + 82  [0x10cee0452]  unsafe_cell.rs:14
    +                                                               2427 tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::_$u7b$$u7b$closure$u7d$$u7d$::hc6bb476f4c0dbfdd  (in cloudfront_logs) + 331  [0x10cebadbb]  core.rs:173
    +                                                                 2427 _$LT$tokio..runtime..blocking..task..BlockingTask$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::hf1bc8661f8dd08ef  (in cloudfront_logs) + 200  [0x10ceac368]  task.rs:38
    +                                                                   2427 _$LT$hyper..client..connect..dns..GaiResolver$u20$as$u20$tower_service..Service$LT$hyper..client..connect..dns..Name$GT$$GT$::call::_$u7b$$u7b$closure$u7d$$u7d$::h97f68211e6cfbc52  (in cloudfront_logs) + 489  [0x10cecac69]  dns.rs:122
    +                                                                     2427 _$LT$$LP$$RF$str$C$u16$RP$$u20$as$u20$std..net..addr..ToSocketAddrs$GT$::to_socket_addrs::hb8b02ee58d460452  (in cloudfront_logs) + 261  [0x10d1dea85]  addr.rs:963
    +                                                                       2427 _$LT$std..sys_common..net..LookupHost$u20$as$u20$core..convert..TryFrom$LT$$LP$$RF$str$C$u16$RP$$GT$$GT$::try_from::hf4c7dff3e0f3ab3c  (in cloudfront_logs) + 301  [0x10d1e5cdd]  net.rs:197
    +                                                                         2427 getaddrinfo  (in libsystem_info.dylib) + 61  [0x7fff63b28e7d]
    +                                                                           2427 _getaddrinfo_internal  (in libsystem_info.dylib) + 231  [0x7fff63b28f77]
    +                                                                             2427 si_addrinfo  (in libsystem_info.dylib) + 1904  [0x7fff63b29810]
    +                                                                               2427 search_addrinfo  (in libsystem_info.dylib) + 309  [0x7fff63b32d15]
    +                                                                                 2427 mdns_addrinfo  (in libsystem_info.dylib) + 1150  [0x7fff63b331ee]
    +                                                                                   2427 _mdns_search  (in libsystem_info.dylib) + 146  [0x7fff63b2d7d2]
    +                                                                                     2427 _mdns_search_ex  (in libsystem_info.dylib) + 2092  [0x7fff63b2e00c]
    +                                                                                       2427 kevent  (in libsystem_kernel.dylib) + 10  [0x7fff63b7078a]

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions