SM magic assert on callback with EVENT_INTERVAL

[sudheerv]

Looks like an HttpSM assert failure when delaying do_http_server_open() - rescheduled due to ProxyTransaction and netvc not being on the same thread (ProxyTransaction::adjust_thread() logic).

Weird thing is that when I check the magic for HttpSM in the Continuation's handleEvent dispatcher, it seems to be valid.

(gdb) bt
#0  0x00002ac802788207 in raise () from /lib64/libc.so.6
#1  0x00002ac8027898f8 in abort () from /lib64/libc.so.6
#2  0x00002ac7ffe4835b in ink_abort (message_format=message_format@entry=0x2ac7ffec52a7 "%s:%d: failed assertion `%s`") at ink_error.cc:99
#3  0x00002ac7ffe45045 in _ink_assert (expression=expression@entry=0x7de8f0 "magic == HTTP_SM_MAGIC_ALIVE", file=file@entry=0x7de2f6 "HttpSM.cc", line=line@entry=2701) at ink_assert.cc:37
#4  0x000000000055069a in HttpSM::main_handler (this=<optimized out>, event=<optimized out>, data=<optimized out>) at HttpSM.cc:2701
#5  0x00000000007bb7c0 in handleEvent (data=0x2ac846a9a7c0, event=2, this=0x2ac8926a4420) at I_Continuation.h:193
#6  EThread::process_event (this=this@entry=0x2ac806513000, e=e@entry=0x2ac846a9a7c0, calling_code=2) at UnixEThread.cc:127
#7  0x00000000007bc834 in EThread::execute_regular (this=this@entry=0x2ac806513000) at UnixEThread.cc:236
#8  0x00000000007bcc62 in EThread::execute (this=0x2ac806513000) at UnixEThread.cc:327
#9  0x00000000007bafe9 in spawn_thread_internal (a=0x2ac803d63c40) at Thread.cc:92
#10 0x00002ac801a9edd5 in start_thread () from /lib64/libpthread.so.0
#11 0x00002ac80284fead in clone () from /lib64/libc.so.6
(gdb) f 5
#5  0x00000000007bb7c0 in handleEvent (data=0x2ac846a9a7c0, event=2, this=0x2ac8926a4420) at I_Continuation.h:193
193	I_Continuation.h: No such file or directory.
(gdb) p event
$21 = 2
(gdb) p *this
$22 = {<force_VFPT_to_top> = {_vptr.force_VFPT_to_top = 0x7e0110 <vtable for HttpSM+24>}, handler = (int (Continuation::*)(Continuation * const, int, void *)) 0x550670 <HttpSM::main_handler(int, void*)>, mutex = {
    m_ptr = 0x2ac8a11439b0}, link = {<SLink<Continuation>> = {next = 0x0}, prev = 0x0}, control_flags = {raw_flags = 0}, thread_affinity = 0x2ac805406000}
(gdb) p ((HttpSM*)0x2ac8926a4420)->magic
$23 = 65261
(gdb) p ((HttpSM*)0x2ac8926a4420)->default_handler
$24 = (int (HttpSM::*)(HttpSM * const, int, void *)) 0x555bd0 <HttpSM::state_http_server_open(int, void*)>
(gdb) 
(gdb) p ((HttpSM*)0x2ac8926a4420)->t_state.m_magic
$26 = HttpTransact::HTTP_TRANSACT_MAGIC_ALIVE

[shinrich]

Is the HttpSM in your core not reallocated, so there might be something interesting in the history member?

[sudheerv]

Below is the history and some other relevant context from SM. It's pretty weird because in the frame 5 (handleEvent) when I cast the continuation to HttpSM, everything (including the magic) seems fine. Yet, the subsequent frame trips the assert in HttpSM::main_handler for this->magic. I can't seem to print anything from the HttpSM::main_handler frame as all the data members seem to be optimized in that frame in gdb.

(gdb) p ((HttpSM*)this)->default_handler
$8 = (int (HttpSM::*)(HttpSM * const, int, void *)) 0x555bd0 <HttpSM::state_http_server_open(int, void*)>
(gdb) p ((HttpSM*)this)->t_state.next_action
$9 = HttpTransact::SM_ACTION_ORIGIN_SERVER_OPEN
(gdb) p ((HttpSM*)this)->t_state.cdn_saved_next_action
$10 = HttpTransact::SM_ACTION_ORIGIN_SERVER_OPEN
(gdb) p ((HttpSM*)this)->t_state.cache_info.action    
$11 = HttpTransact::CACHE_DO_NO_ACTION
(gdb) p ((HttpSM*)this)->server_entry             
$12 = (HttpVCTableEntry *) 0x2ac8926a5868
(gdb) p *((HttpSM*)this)->server_entry
$13 = {vc = 0x2ac8e45837d0, read_buffer = 0x0, write_buffer = 0x0, read_vio = 0x2ac8b9a47660, write_vio = 0x2ac8b9a476d0, vc_handler = (int (HttpSM::*)(HttpSM * const, int, void *)) 0x555bd0 <HttpSM::state_http_server_open(int, void*)>, vc_type = HTTP_SERVER_VC, sm = 0xacb7a8 <httpSMAllocator+8>, 
  eos = false, in_tunnel = false}
(gdb) p *((HttpSM*)this)->pending_action
Cannot access memory at address 0x0
(gdb) p ((HttpSM*)this)->pending_action
$14 = (Action *) 0x0

(gdb) p ((HttpSM*)this)->vc_table
$15 = {static vc_table_max_entries = 4, vc_table = {{vc = 0x2ac8c7c212c0, read_buffer = 0x0, write_buffer = 0x0, read_vio = 0x2ac849281d90, write_vio = 0x0, vc_handler = (int (HttpSM::*)(HttpSM * const, int, void *)) 0x53e5f0 <HttpSM::state_watch_for_client_abort(int, void*)>, 
      vc_type = HTTP_UA_VC, sm = 0xacb7a8 <httpSMAllocator+8>, eos = false, in_tunnel = false}, {vc = 0x2ac8e45837d0, read_buffer = 0x0, write_buffer = 0x0, read_vio = 0x2ac8b9a47660, write_vio = 0x2ac8b9a476d0, vc_handler = (int (HttpSM::*)(HttpSM * const, int, 
    void *)) 0x555bd0 <HttpSM::state_http_server_open(int, void*)>, vc_type = HTTP_SERVER_VC, sm = 0xacb7a8 <httpSMAllocator+8>, eos = false, in_tunnel = false}, {vc = 0x0, read_buffer = 0x0, write_buffer = 0x0, read_vio = 0x0, write_vio = 0x0, vc_handler = NULL, vc_type = HTTP_UNKNOWN, sm = 0x0, 
      eos = false, in_tunnel = false}, {vc = 0x0, read_buffer = 0x0, write_buffer = 0x0, read_vio = 0x0, write_vio = 0x0, vc_handler = NULL, vc_type = HTTP_UNKNOWN, sm = 0x0, eos = false, in_tunnel = false}}, sm = 0xacb7a8 <httpSMAllocator+8>}
(gdb) set print pretty
(gdb) p ((HttpSM*)this)->vc_table
$16 = {
  static vc_table_max_entries = 4, 
  vc_table = {{
      vc = 0x2ac8c7c212c0, 
      read_buffer = 0x0, 
      write_buffer = 0x0, 
      read_vio = 0x2ac849281d90, 
      write_vio = 0x0, 
      vc_handler = (int (HttpSM::*)(HttpSM * const, int, void *)) 0x53e5f0 <HttpSM::state_watch_for_client_abort(int, void*)>, 
      vc_type = HTTP_UA_VC, 
      sm = 0xacb7a8 <httpSMAllocator+8>, 
      eos = false, 
      in_tunnel = false
    }, {
      vc = 0x2ac8e45837d0, 
      read_buffer = 0x0, 
      write_buffer = 0x0, 
      read_vio = 0x2ac8b9a47660, 
      write_vio = 0x2ac8b9a476d0, 
      vc_handler = (int (HttpSM::*)(HttpSM * const, int, void *)) 0x555bd0 <HttpSM::state_http_server_open(int, void*)>, 
      vc_type = HTTP_SERVER_VC, 
      sm = 0xacb7a8 <httpSMAllocator+8>, 
      eos = false, 
      in_tunnel = false
    }, {
      vc = 0x0, 
      read_buffer = 0x0, 
      write_buffer = 0x0, 
      read_vio = 0x0, 
      write_vio = 0x0, 
      vc_handler = NULL, 
      vc_type = HTTP_UNKNOWN, 
      sm = 0x0, 
      eos = false, 
      in_tunnel = false
    }, {
      vc = 0x0, 
      read_buffer = 0x0, 
      write_buffer = 0x0, 
      read_vio = 0x0, 
      write_vio = 0x0, 
      vc_handler = NULL, 
      vc_type = HTTP_UNKNOWN, 
      sm = 0x0, 
      eos = false, 
      in_tunnel = false
    }}, 
  sm = 0xacb7a8 <httpSMAllocator+8>
}





(gdb) up
#5  0x00000000007bb7c0 in handleEvent (data=0x2ac846a9a7c0, event=2, this=0x2ac8926a4420) at I_Continuation.h:193
193	I_Continuation.h: No such file or directory.
(gdb) p ((HttpSM*)this)->history
$19 = {
  history = {{
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dff00 <HttpSM::state_add_to_list(int, void*)::__FUNCTION__> "state_add_to_list", 
        line = 379
      }, 
      event = 0, 
      reentrancy = 1
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfe80 <HttpSM::state_read_client_request_header(int, void*)::__FUNCTION__> "state_read_client_request_header", 
        line = 635
      }, 
      event = 100, 
      reentrancy = 2
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfe80 <HttpSM::state_read_client_request_header(int, void*)::__FUNCTION__> "state_read_client_request_header", 
        line = 635
      }, 
      event = 100, 
      reentrancy = 1
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfe80 <HttpSM::state_read_client_request_header(int, void*)::__FUNCTION__> "state_read_client_request_header", 
        line = 635
      }, 
      event = 100, 
      reentrancy = 1
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 2
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 2
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 3
    }, {
      location = {
---Type <return> to continue, or q <return> to quit---
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 3
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 4
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 4
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 5
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 5
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 6
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 6
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
---Type <return> to continue, or q <return> to quit---
      }, 
      event = 60000, 
      reentrancy = 7
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 7
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 1
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 1
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 2
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 2
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 1
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 1
---Type <return> to continue, or q <return> to quit---
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 2
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 2
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 3
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 3
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 4
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 4
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 5
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
---Type <return> to continue, or q <return> to quit---
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 5
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 6
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 6
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 7
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 7
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 8
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 8
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
---Type <return> to continue, or q <return> to quit---
      event = 60000, 
      reentrancy = 9
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 9
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 10
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 10
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 11
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 11
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 12
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 12
    }, {
---Type <return> to continue, or q <return> to quit---
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 13
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 13
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 14
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 14
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 15
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 15
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 16
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
---Type <return> to continue, or q <return> to quit---
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 16
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 17
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 17
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 18
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 18
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
      reentrancy = 19
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 19
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfd10 <HttpSM::state_api_callback(int, void*)::__FUNCTION__> "state_api_callback", 
        line = 1364
      }, 
      event = 60000, 
---Type <return> to continue, or q <return> to quit---
      reentrancy = 20
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfcf0 <HttpSM::state_api_callout(int, void*)::__FUNCTION__> "state_api_callout", 
        line = 1404
      }, 
      event = 60000, 
      reentrancy = 20
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7df218 <HttpSM::set_next_state()::__FUNCTION__> "set_next_state", 
        line = 7486
      }, 
      event = 34463, 
      reentrancy = 20
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7df218 <HttpSM::set_next_state()::__FUNCTION__> "set_next_state", 
        line = 7532
      }, 
      event = 34463, 
      reentrancy = 20
    }, {
      location = {
        file = 0x7de2f6 "HttpSM.cc", 
        func = 0x7dfc90 <HttpSM::state_http_server_open(int, void*)::__FUNCTION__> "state_http_server_open", 
        line = 1755
      }, 
      event = 200, 
      reentrancy = 21
    }, {
      location = {
        file = 0x0, 
        func = 0x0, 
        line = 0
      }, 
      event = 0, 
      reentrancy = 0
    }, {
      location = {
        file = 0x0, 
        func = 0x0, 
        line = 0
      }, 
      event = 0, 
      reentrancy = 0
    }}, 
  history_pos = 63
}

[sudheerv]

Reverted PR 4028 as that seems to be related. Running the reverted version now in prod. Will keep posted on how it goes.

[sudheerv]

Reverting PR 4028 does seem to have fixed this problem

[sudheerv]

#6889 fixes this