Skip to content

Spinning thread taking 100% cpu in http/2 dependency tree #3990

Description

@bryancall
  47.61%  traffic_server         [.] Http2DependencyTree::Tree<Http2Stream*>::update
  46.37%  traffic_server         [.] PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down
   0.28%  [kernel]               [k] vsnprintf
   0.27%  [kernel]               [k] format_decode


-   84.36%    43.15%  traffic_server         [.] Http2DependencyTree::Tree<Http2Stream*>::update                                                                                                                                             ▒
   + 43.15% start_thread                                                                                                                                                                                                                     ▒
   - 41.20% Http2DependencyTree::Tree<Http2Stream*>::update                                                                                                                                                                                  ▒
        PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down



root@qa1 262/0 # pstack $(pidof traffic_server)
Thread 23 (Thread 0x2b3a3592a700 (LWP 14836)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000e4697e in ink_cond_timedwait (t=0x2b3a35929ad0, mp=0x2b3a35203330, cp=0x2b3a35203358) at ../../lib/ts/ink_thread.h:259
#2  ProtectedQueue::wait (this=0x2b3a35203310, timeout=<optimized out>) at ProtectedQueue.cc:120
#3  0x0000000000e4c2ad in EThread::DefaultTailHandler::waitForActivity (timeout=60000000, this=<optimized out>) at I_EThread.h:368
#4  EThread::execute_regular (this=0x2b3a35102800) at UnixEThread.cc:272
#5  0x0000000000e47630 in spawn_thread_internal (a=0x60600005d320) at Thread.cc:85
#6  0x00002b3a2514be25 in start_thread (arg=0x2b3a3592a700) at pthread_create.c:308
#7  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 22 (Thread 0x2b3a35713700 (LWP 14835)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000e4697e in ink_cond_timedwait (t=0x2b3a35712ad0, mp=0x2b3a346e5330, cp=0x2b3a346e5358) at ../../lib/ts/ink_thread.h:259
#2  ProtectedQueue::wait (this=0x2b3a346e5310, timeout=<optimized out>) at ProtectedQueue.cc:120
#3  0x0000000000e4c2ad in EThread::DefaultTailHandler::waitForActivity (timeout=60000000, this=<optimized out>) at I_EThread.h:368
#4  EThread::execute_regular (this=0x2b3a345e4800) at UnixEThread.cc:272
#5  0x0000000000e47630 in spawn_thread_internal (a=0x60600005d260) at Thread.cc:85
#6  0x00002b3a2514be25 in start_thread (arg=0x2b3a35713700) at pthread_create.c:308
#7  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 21 (Thread 0x2b3a2ddce700 (LWP 14834)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2ddcdc60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dd10) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2dbc0800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038a80) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2ddce700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 20 (Thread 0x2b3a2dba8700 (LWP 14833)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2dba7c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dca0) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d99a800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038a20) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2dba8700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 19 (Thread 0x2b3a2d982700 (LWP 14832)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d981c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dc30) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d774800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000389c0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d982700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 18 (Thread 0x2b3a2d75c700 (LWP 14831)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d75bc60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dbc0) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d54e800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038960) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d75c700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 17 (Thread 0x2b3a2d536700 (LWP 14830)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d535c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000db50) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d328800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038900) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d536700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 16 (Thread 0x2b3a2d310700 (LWP 14829)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d30fc60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dae0) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d102800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000388a0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d310700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 15 (Thread 0x2b3a2d0ea700 (LWP 14828)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d0e9c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000da70) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2cedc800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038840) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d0ea700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 14 (Thread 0x2b3a2cec4700 (LWP 14825)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2cec3c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000da00) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2ccb6800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000387e0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2cec4700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 13 (Thread 0x2b3a2cc9e700 (LWP 14823)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000e4697e in ink_cond_timedwait (t=0x2b3a2cc9dad0, mp=0x2b3a2ca91330, cp=0x2b3a2ca91358) at ../../lib/ts/ink_thread.h:259
#2  ProtectedQueue::wait (this=0x2b3a2ca91310, timeout=<optimized out>) at ProtectedQueue.cc:120
#3  0x0000000000e4c2ad in EThread::DefaultTailHandler::waitForActivity (timeout=60000000, this=<optimized out>) at I_EThread.h:368
#4  EThread::execute_regular (this=0x2b3a2c990800) at UnixEThread.cc:272
#5  0x0000000000e47630 in spawn_thread_internal (a=0x6060000386c0) at Thread.cc:85
#6  0x00002b3a2514be25 in start_thread (arg=0x2b3a2cc9e700) at pthread_create.c:308
#7  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 12 (Thread 0x2b3a2c274700 (LWP 14822)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00002b3a2358e2a0 in EventNotify::wait (this=0x602000015cb0) at EventNotify.cc:88
#2  0x000000000084ad7a in Log::flush_thread_main () at Log.cc:1373
#3  0x00000000008550bd in LoggingFlushContinuation::mainEvent (this=<optimized out>) at Log.cc:290
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2c066800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000352a0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2c274700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 11 (Thread 0x2b3a2c04e700 (LWP 14821)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00002b3a2358e2a0 in EventNotify::wait (this=0x602000015c98) at EventNotify.cc:88
#2  0x000000000084a013 in Log::preproc_thread_main (args=<optimized out>) at Log.cc:1256
#3  0x000000000085509d in LoggingPreprocContinuation::mainEvent (this=<optimized out>) at Log.cc:277
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2be40800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000351e0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2c04e700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 10 (Thread 0x2b3a2bc48700 (LWP 14820)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a6db4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a6d7800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000016460) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2bc48700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 9 (Thread 0x2b3a2b98e700 (LWP 14819)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a5cc4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a5c8800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x6060000163a0) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b98e700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 8 (Thread 0x2b3a2b6d4700 (LWP 14818)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a4bd4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a4b9800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x6060000162e0) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b6d4700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 7 (Thread 0x2b3a2b41a700 (LWP 14817)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a3ae4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a3aa800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000016220) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b41a700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 6 (Thread 0x2b3a2b160700 (LWP 14816)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a29f4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a29b800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000016160) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b160700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 5 (Thread 0x2b3a2aea6700 (LWP 14815)):
#0  0x000000000083fedf in std::vector<PriorityQueueEntry<Http2DependencyTree::Node*>*, std::allocator<PriorityQueueEntry<Http2DependencyTree::Node*>*> >::empty (this=0x60300007cd20) at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/stl_vector.h:760
#1  PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::empty (this=0x60300007cd20) at ../../lib/ts/PriorityQueue.h:92
#2  PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down (this=0x60300007cd20, index=<optimized out>) at ../../lib/ts/PriorityQueue.h:234
#3  0x000000000082fcb1 in PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::update (increased=true, entry=<optimized out>, this=<optimized out>) at ../../lib/ts/PriorityQueue.h:189
#4  Http2DependencyTree::Tree<Http2Stream*>::update (node=0x60700001fac0, node@entry=0x60700001fa50, sent=sent@entry=0, this=<optimized out>) at Http2DependencyTree.h:422
#5  0x0000000000830d41 in Http2DependencyTree::Tree<Http2Stream*>::deactivate (this=<optimized out>, sent=0, node=<optimized out>) at Http2DependencyTree.h:411
#6  Http2ConnectionState::delete_stream (this=this@entry=0x2b3a36210e30, stream=stream@entry=0x2b3a36233800) at Http2ConnectionState.cc:1192
#7  0x0000000000832976 in Http2ConnectionState::cleanup_streams (this=this@entry=0x2b3a36210e30) at Http2ConnectionState.cc:1158
#8  0x0000000000834d8d in Http2ConnectionState::main_event_handler (this=0x2b3a36210e30, event=<optimized out>, edata=<optimized out>) at Http2ConnectionState.cc:897
#9  0x000000000080f195 in Continuation::handleEvent (data=<optimized out>, event=<optimized out>, this=0x2b3a36210e30) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
#10 send_connection_event (cont=cont@entry=0x2b3a36210e30, event=event@entry=2252, edata=edata@entry=0x2b3a36210b60) at Http2ClientSession.cc:58
#11 0x000000000081416f in Http2ClientSession::do_io_close (this=0x2b3a36210b60, alerrno=<optimized out>) at Http2ClientSession.cc:261
#12 0x000000000081231a in Http2ClientSession::main_event_handler (this=0x2b3a36210b60, event=<optimized out>, edata=0x63400005ed90) at Http2ClientSession.cc:334
#13 0x0000000000d64796 in Continuation::handleEvent (data=0x63400005ed90, event=<optimized out>, this=<optimized out>) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
#14 read_signal_and_update (event=<optimized out>, vc=0x63400005ec40) at UnixNetVConnection.cc:83
#15 0x0000000000d67ba7 in UnixNetVConnection::mainEvent (this=0x63400005ec40, event=<optimized out>, e=<optimized out>) at UnixNetVConnection.cc:1189
#16 0x0000000000d36395 in Continuation::handleEvent (data=0x62c000006b20, event=1, this=0x63400005ec40) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
#17 InactivityCop::check_inactivity (this=<optimized out>, event=<optimized out>, e=<optimized out>) at UnixNet.cc:81
#18 0x0000000000e48bb0 in Continuation::handleEvent (data=0x62c000006b20, event=2, this=<optimized out>) at I_Continuation.h:160
#19 EThread::process_event (this=this@entry=0x2b3a2a18c800, e=e@entry=0x62c000006b20, calling_code=<optimized out>) at UnixEThread.cc:131
#20 0x0000000000e4bec6 in EThread::execute_regular (this=0x2b3a2a18c800) at UnixEThread.cc:244
#21 0x0000000000e47630 in spawn_thread_internal (a=0x6060000160a0) at Thread.cc:85
#22 0x00002b3a2514be25 in start_thread (arg=0x2b3a2aea6700) at pthread_create.c:308
#23 0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 4 (Thread 0x2b3a2abf4700 (LWP 14814)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a0814c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a07d800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000015fe0) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2abf4700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 3 (Thread 0x2b3a2a9e5700 (LWP 14813)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a29f724c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a29f6e800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000015f20) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2a9e5700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 2 (Thread 0x2b3a29a00700 (LWP 14810)):
#0  0x00002b3a25e78c73 in select () at ../sysdeps/unix/syscall-template.S:81
#1  0x00000000009bc234 in mgmt_select (nfds=nfds@entry=1024, readfds=readfds@entry=0x2b3a299ffc30, writefds=writefds@entry=0x0, errorfds=errorfds@entry=0x0, timeout=timeout@entry=0x2b3a299ffb70) at MgmtSocket.cc:195
#2  0x00000000009b1b90 in ProcessManager::pollLMConnection (this=0x60c000002b00) at ProcessManager.cc:373
#3  0x00000000009b276e in ProcessManager::processManagerThread (arg=<optimized out>) at ProcessManager.cc:169
#4  0x00002b3a2514be25 in start_thread (arg=0x2b3a29a00700) at pthread_create.c:308
#5  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 1 (Thread 0x2b3a2236d040 (LWP 14809)):
#0  0x00002b3a25e4856d in nanosleep () at ../sysdeps/unix/syscall-template.S:81
#1  0x00002b3a25e48404 in __sleep (seconds=0, seconds@entry=1) at ../sysdeps/unix/sysv/linux/sleep.c:137
#2  0x00000000004b17f3 in main (argv=<optimized out>) at traffic_server/traffic_server.cc:2013


(gdb) thread apply 5 bt full
Thread 5 (Thread 0x2b3a2aea6700 (LWP 14815)):
#0  0x000000000083ff9e in PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down (this=0x60300007ccc0, index=0) at ../../lib/ts/PriorityQueue.h:244
        left = 1
        right = <optimized out>
        smaller = <optimized out>
#1  0x000000000082fcb1 in PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::update (increased=true, entry=<optimized out>, this=<optimized out>) at ../../lib/ts/PriorityQueue.h:189
        increased = true
#2  Http2DependencyTree::Tree<Http2Stream*>::update (node=0x60700001fba0, node@entry=0x60700001fa50, sent=sent@entry=0, this=<optimized out>) at Http2DependencyTree.h:422
No locals.
#3  0x0000000000830d41 in Http2DependencyTree::Tree<Http2Stream*>::deactivate (this=<optimized out>, sent=0, node=<optimized out>) at Http2DependencyTree.h:411
No locals.
#4  Http2ConnectionState::delete_stream (this=this@entry=0x2b3a36210e30, stream=stream@entry=0x2b3a36233800) at Http2ConnectionState.cc:1192
        node = 0x60700001fa50
        __FUNCTION__ = "delete_stream"
#5  0x0000000000832976 in Http2ConnectionState::cleanup_streams (this=this@entry=0x2b3a36210e30) at Http2ConnectionState.cc:1158
        next = 0x2b3a36233c00
        s = 0x2b3a36233800
#6  0x0000000000834d8d in Http2ConnectionState::main_event_handler (this=0x2b3a36210e30, event=<optimized out>, edata=<optimized out>) at Http2ConnectionState.cc:897
        __FUNCTION__ = "main_event_handler"
#7  0x000000000080f195 in Continuation::handleEvent (data=<optimized out>, event=<optimized out>, this=0x2b3a36210e30) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
No locals.
#8  send_connection_event (cont=cont@entry=0x2b3a36210e30, event=event@entry=2252, edata=edata@entry=0x2b3a36210b60) at Http2ClientSession.cc:58
No locals.
#9  0x000000000081416f in Http2ClientSession::do_io_close (this=0x2b3a36210b60, alerrno=<optimized out>) at Http2ClientSession.cc:261
        __FUNCTION__ = "do_io_close"
#10 0x000000000081231a in Http2ClientSession::main_event_handler (this=0x2b3a36210b60, event=<optimized out>, edata=0x63400005ed90) at Http2ClientSession.cc:334
        retval = <optimized out>
        e = 0x63400005ed90
        __FUNCTION__ = "main_event_handler"
#11 0x0000000000d64796 in Continuation::handleEvent (data=0x63400005ed90, event=<optimized out>, this=<optimized out>) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
No locals.
#12 read_signal_and_update (event=<optimized out>, vc=0x63400005ec40) at UnixNetVConnection.cc:83
No locals.
#13 0x0000000000d67ba7 in UnixNetVConnection::mainEvent (this=0x63400005ec40, event=<optimized out>, e=<optimized out>) at UnixNetVConnection.cc:1189
        signal_event = 105
        reader_cont = 0x2b3a36210b60
        writer_cont = <optimized out>
        signal_timeout_at = 0x63400005ee98
        t = 0x0
        signal_timeout = <synthetic pointer>
#14 0x0000000000d36395 in Continuation::handleEvent (data=0x62c000006b20, event=1, this=0x63400005ec40) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
No locals.
#15 InactivityCop::check_inactivity (this=<optimized out>, event=<optimized out>, e=<optimized out>) at UnixNet.cc:81
        vc = 0x63400005ec40
        __FUNCTION__ = "check_inactivity"
#16 0x0000000000e48bb0 in Continuation::handleEvent (data=0x62c000006b20, event=2, this=<optimized out>) at I_Continuation.h:160
No locals.
#17 EThread::process_event (this=this@entry=0x2b3a2a18c800, e=e@entry=0x62c000006b20, calling_code=<optimized out>) at UnixEThread.cc:131
        c_temp = <optimized out>
#18 0x0000000000e4bec6 in EThread::execute_regular (this=0x2b3a2a18c800) at UnixEThread.cc:244
        done_one = true
        sleep_time = <optimized out>
        e = 0x62c000006b20
        next_time = <optimized out>
        delta = <optimized out>
        prev_metric = 0x2b3a2a290f30
        METRIC_INIT = {
          _loop_time = {
            _start = 0,
            _min = 9223372036854775807,
            _max = 0
          },
          _events = {
            _min = 2147483647,
            _max = 0,
            _total = 0
          },
          _count = 0,
          _wait = 0
        }
#19 0x0000000000e47630 in spawn_thread_internal (a=0x6060000160a0) at Thread.cc:85
        p = 0x6060000160a0
#20 0x00002b3a2514be25 in start_thread (arg=0x2b3a2aea6700) at pthread_create.c:308
        __res = <optimized out>
        pd = 0x2b3a2aea6700
        now = <optimized out>
        unwind_buf = {
          cancel_jmp_buf = {{
              jmp_buf = {47528828102400, -6944238515488047313, 0, 2101248, 0, 47528828102400, -3903127703691215057, -3903123386193821905},
              mask_was_saved = 0
            }},
          priv = {
            pad = {0x0, 0x0, 0x0, 0x0},
            data = {
              prev = 0x0,
              cleanup = 0x0,
              canceltype = 0
            }
          }
        }
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
#21 0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
No locals.

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions