Crash in write_to_net_io in 8.1.x

[zwoop]

This is with current 8.1.x branch:

#1  0x000055ebc0c7a064 in crash_logger_invoke (signo=11, info=0x7f52d950b0f0, ctx=0x7f52d950afc0) at traffic_server/Crash.cc:165
        status = 0
#2  <signal handler called>
No symbol table info available.
#3  0x00000000e27c2211 in ?? ()
No symbol table info available.
#4  0x000055ebc0e6f94a in write_to_net_io (nh=0x7f52dea119e0, vc=0x7f47411a3c00, thread=0x7f52dea0dd00) at UnixNetVConnection.cc:390
        ret = <optimized out>
        err = <optimized out>
        mutex = 0x7f52e2d53710
        lock = {
          m = {
            m_ptr = 0x7f47e0116640
          },
          lock_acquired = true
        }
        ntodo = <optimized out>
        buf = <optimized out>
        towrite = <optimized out>
        signalled = <optimized out>
        needs = <optimized out>
        total_written = <optimized out>
#5  0x000055ebc0e692ab in NetHandler::process_ready_list (this=0x7f52dea119e0) at UnixNet.cc:416
No locals.
#6  0x000055ebc0e699df in NetHandler::waitForActivity (this=<optimized out>, timeout=<optimized out>) at UnixNet.cc:532
        pd = 0x7f52d8c0ff40
        epd = <optimized out>
        vc = <optimized out>
        lock = <optimized out>
#7  0x000055ebc0e69aad in non-virtual thunk to NetHandler::waitForActivity(long) ()
        stat_prefix = <incomplete type>
        NetHandler::config_value_affects_per_thread_value = {
          <std::__1::__bitset<1, 6>> = {
            __first_ = 3
          },
        }
        last_shedding_warning = 0
        fds_limit = 1000000
        net_connections_throttle = 100000
        last_throttle_warning = 0
        fds_throttle = 100000
        last_transient_accept_error = 0
        NetHandler::active_thread_types = {
          <std::__1::__bitset<1, 32>> = {
            __first_ = 3
          },
        }
        NetHandler::global_config = {
          max_connections_in = 75000,
          max_connections_active_in = 30000,
          inactive_threshold_in = 0,
          transaction_no_activity_timeout_in = 0,
          keep_alive_no_activity_timeout_in = 0,
          default_inactivity_timeout = 900
        }
        net_memory_throttle = false
#8  0x000055ebc0e984f1 in EThread::execute_regular (this=0x7f52dea0dd00) at UnixEThread.cc:278
        sleep_time = 60000000
        done_one = <optimized out>
        METRIC_INIT = {
          _loop_time = {
            _start = 0,
            _min = 9223372036854775807,
            _max = 0
          },
          _events = {
            _min = 2147483647,
            _max = 0,
            _total = 0
          },
          _count = 0,
          _wait = 0
        }
        delta = 0
        next_time = <optimized out>
        nq_count = 2
        prev_metric = 0x7f52deb1a4a0
        ev_count = 2
        loop_start_time = <optimized out>
        loop_finish_time = <optimized out>
#9  0x000055ebc0e9718c in spawn_thread_internal (a=0x7f52e3a37350) at Thread.cc:85
        p = 0x7f52e3a37350
#10 0x00007f52e5343e65 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#11 0x00007f52e476188d in clone () from /lib64/libc.so.6
No symbol table info available.

Frame #4 is

(gdb) p vc
$4 = (UnixNetVConnection *) 0x7f47411a3c00
(gdb) p *vc
$5 = {
  <NetVConnection> = {
    <AnnotatedVConnection> = {
      <VConnection> = {
        <Continuation> = {
          <force_VFPT_to_top> = {
            _vptr$force_VFPT_to_top = 0x7f4741155400
          },
          members of Continuation:
          handler = (int (Continuation::*)(Continuation * const, int, void *)) 0x55ebc0e728a0 <UnixNetVConnection::startEvent(int, Event*)>,
          mutex = {
            m_ptr = 0x0
          },
          link = {
            <SLink<Continuation>> = {
              next = 0x0
            },
            members of Link<Continuation>:
            prev = 0x0
          },
          control_flags = {
            raw_flags = 0
          }
        },
        members of VConnection:
        lerrno = 0
      },
      members of AnnotatedVConnection:
      user_args = {
        __elems_ = {0x0, 0x0, 0x0, 0x0}
      }
    },
    members of NetVConnection:
    options = {
      ip_proto = NetVCOptions::USE_TCP,
      ip_family = 2,
      local_ip = {
        _family = 0,
        _addr = {
          _ip4 = 3645944920,
          _ip6 = {
            __in6_u = {
              __u6_addr8 = "X\264P\331R\177\000\000,P\354\345R\177\000",
              __u6_addr16 = {46168, 55632, 32594, 0, 20524, 58860, 32594, 0},
              __u6_addr32 = {3645944920, 32594, 3857469484, 32594}
            }
          },
          _byte = "X\264P\331R\177\000\000,P\354\345R\177\000",
          _u32 = {3645944920, 32594, 3857469484, 32594},
          _u64 = {139993809990744, 139994021515308}
        }
      },
      local_port = 0,
      addr_binding = NetVCOptions::ANY_ADDR,
      f_blocking = false,
      f_blocking_connect = false,
      f_tcp_fastopen = false,
      socks_support = 0 '\000',
      socks_version = 0 '\000',
      socket_recv_bufsize = 0,
      socket_send_bufsize = 0,
      sockopt_flags = 0,
      packet_mark = 0,
      packet_tos = 0,
      etype = 0,
      sni_servername = {
        <ats_scoped_resource<detail::SCOPED_MALLOC_TRAITS<char> >> = {
          _r = 0x0
        }, <No data fields>},
      ssl_servername = {
        <ats_scoped_resource<detail::SCOPED_MALLOC_TRAITS<char> >> = {
          _r = 0x0
        }, <No data fields>},
      clientCertificate = {
        <ats_scoped_resource<detail::SCOPED_MALLOC_TRAITS<char> >> = {
          _r = 0x0
        }, <No data fields>},
      clientVerificationFlag = 0 '\000'
    },
    socks_addr = {
      type = 0 '\000',
      addr = {
        ipv4 = "\000\000\000",
        buf = 0x0
      }
    },
    attributes = 0,
    thread = 0x7f52dea0dd00,
    pp_info = {
      proxy_protocol_version = NetVConnection::UNDEFINED,
      ip_family = 0,
      src_addr = {
        sa = {
          sa_family = 0,
          sa_data = '\000' <repeats 13 times>
        },
        sin = {
          sin_family = 0,
          sin_port = 0,
          sin_addr = {
            s_addr = 0
          },
          sin_zero = "\000\000\000\000\000\000\000"
        },
        sin6 = {
          sin6_family = 0,
          sin6_port = 0,
          sin6_flowinfo = 0,
          sin6_addr = {
            __in6_u = {
              __u6_addr8 = '\000' <repeats 15 times>,
              __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0},
              __u6_addr32 = {0, 0, 0, 0}
            }
          },
          sin6_scope_id = 0
        }
      },
      dst_addr = {
        sa = {
          sa_family = 0,
          sa_data = '\000' <repeats 13 times>
        },
        sin = {
          sin_family = 0,
          sin_port = 0,
          sin_addr = {
            s_addr = 0
          },
          sin_zero = "\000\000\000\000\000\000\000"
        },
        sin6 = {
          sin6_family = 0,
          sin6_port = 0,
          sin6_flowinfo = 0,
          sin6_addr = {
            __in6_u = {
              __u6_addr8 = '\000' <repeats 15 times>,
              __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0},
              __u6_addr32 = {0, 0, 0, 0}
            }
          },
          sin6_scope_id = 0
        }
      }
    },
    local_addr = {
      sa = {
        sa_family = 2,
        sa_data = "\276\350\021\375\001\205\000\000\000\000\000\000\000"
      },
      sin = {
        sin_family = 2,
        sin_port = 59582,
        sin_addr = {
          s_addr = 2231500049
        },
        sin_zero = "\000\000\000\000\000\000\000"
      },
      sin6 = {
        sin6_family = 2,
        sin6_port = 59582,
        sin6_flowinfo = 2231500049,
        sin6_addr = {
          __in6_u = {
            __u6_addr8 = '\000' <repeats 15 times>,
            __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0},
            __u6_addr32 = {0, 0, 0, 0}
          }
        },
        sin6_scope_id = 0
      }
    },
    remote_addr = {
      sa = {
        sa_family = 0,
        sa_data = '\000' <repeats 13 times>
      },
      sin = {
        sin_family = 0,
        sin_port = 0,
        sin_addr = {
          s_addr = 0
        },
        sin_zero = "\000\000\000\000\000\000\000"
      },
      sin6 = {
        sin6_family = 0,
        sin6_port = 0,
        sin6_flowinfo = 0,
        sin6_addr = {
          __in6_u = {
            __u6_addr8 = '\000' <repeats 15 times>,
            __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0},
            __u6_addr32 = {0, 0, 0, 0}
          }
        },
        sin6_scope_id = 0
      }
    },
    got_local_addr = false,
    got_remote_addr = false,
    is_internal_request = false,
    is_transparent = false,
    is_proxy_protocol = false,
    mptcp_state = {
      <std::__1::__optional_move_assign_base<bool, true>> = {
        <std::__1::__optional_copy_assign_base<bool, true>> = {
          <std::__1::__optional_move_base<bool, true>> = {
            <std::__1::__optional_copy_base<bool, true>> = {
              <std::__1::__optional_storage_base<bool, false>> = {
                <std::__1::__optional_destruct_base<bool, true>> = {
                  {
                    __null_state_ = 0 '\000',
                    __val_ = false
                  },
                  __engaged_ = false
                }, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>},
      <std::__1::__sfinae_ctor_base<true, true>> = {<No data fields>},
      <std::__1::__sfinae_assign_base<true, true>> = {<No data fields>}, <No data fields>},
    write_buffer_empty_event = 0,
    netvc_context = NET_VCONNECTION_UNSET
  },
  members of UnixNetVConnection:
  action_ = {
    _vptr$Action = 0x55ebc1147d38 <vtable for Action+16>,
    continuation = 0x7f47df86e8f0,
    mutex = {
      m_ptr = 0x0
    },
    cancelled = 0
  },
  closed = 0,
  read = {
    enabled = 0,
    vio = {
      cont = 0x0,
      nbytes = 0,
      ndone = 0,
      op = 0,
      buffer = {
        mbuf = 0x0,
        entry = 0x0
      },
      vc_server = 0x0,
      mutex = {
        m_ptr = 0x0
      },
      _disabled = false
    },
    ready_link = {
      <SLink<UnixNetVConnection>> = {
        next = 0x0
      },
      members of Link<UnixNetVConnection>:
      prev = 0x0
    },
    enable_link = {
      next = 0x0
    },
    in_enabled_list = 0,
    triggered = 0
  },
  write = {
    enabled = 0,
    vio = {
      cont = 0x0,
      nbytes = 0,
      ndone = 0,
      op = 0,
      buffer = {
        mbuf = 0x0,
        entry = 0x0
      },
      vc_server = 0x0,
      mutex = {
        m_ptr = 0x0
      },
      _disabled = false
    },
    ready_link = {
      <SLink<UnixNetVConnection>> = {
        next = 0x0
      },
      members of Link<UnixNetVConnection>:
      prev = 0x0
    },
    enable_link = {
      next = 0x0
    },
    in_enabled_list = 0,
    triggered = 0
  },
  cop_link = {
    <SLink<UnixNetVConnection>> = {
      next = 0x0
    },
    members of Link<UnixNetVConnection>:
    prev = 0x0
  },
  keep_alive_queue_link = {
    <SLink<UnixNetVConnection>> = {
      next = 0x0
    },
    members of Link<UnixNetVConnection>:
    prev = 0x0
  },
  active_queue_link = {
    <SLink<UnixNetVConnection>> = {
      next = 0x0
    },
    members of Link<UnixNetVConnection>:
    prev = 0x0
  },
  inactivity_timeout_in = 0,
  active_timeout_in = 0,
  next_inactivity_timeout_at = 0,
  next_activity_timeout_at = 0,
  ep = {
    fd = 1886,
    event_loop = 0x0,
    type = 2,
    data = {
      c = 0x7f47411a3c00,
      vc = 0x7f47411a3c00,
      dnscon = 0x7f47411a3c00,
      na = 0x7f47411a3c00,
      uc = 0x7f47411a3c00
    }
  },
  nh = 0x0,
  id = 1029398,
  {
    flags = 0,
    f = {
      got_local_addr = 0,
      shutdown = 0
    }
  },
  con = {
    _vptr$Connection = 0x55ebc11577f8 <vtable for Connection+16>,
    fd = -1,
    addr = {
      sa = {
        sa_family = 2,
        sa_data = "\001\273\021\274\324\200\000\000\000\000\000\000\000"
      },
      sin = {
        sin_family = 2,
        sin_port = 47873,
        sin_addr = {
          s_addr = 2161425425
        },
        sin_zero = "\000\000\000\000\000\000\000"
      },
      sin6 = {
        sin6_family = 2,
        sin6_port = 47873,
        sin6_flowinfo = 2161425425,
        sin6_addr = {
          __in6_u = {
            __u6_addr8 = '\000' <repeats 15 times>,
            __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0},
            __u6_addr32 = {0, 0, 0, 0}
          }
        },
        sin6_scope_id = 0
      }
    },
    is_bound = false,
    is_connected = false,
    sock_type = 1
  },
  recursion = 0,
  submit_time = 1588068325527879110,
  oob_ptr = 0x0,
  from_accept_thread = false,
  accept_object = 0x0,
  origin_trace = false,
  origin_trace_addr = 0x0,
  origin_trace_port = 0
}

[bryancall]

Looking at this some more, the only way to call write_to_net_io from NetHandler::process_ready_list is for the VC's write.enabled and write.triggered be non-zero.

Looking a the dump of the VC this is not the case. It looks like another thread is either freeing the VC on you or something is overwriting memory.

[masaori335]

@vmamidi found that this is duplicated with #4903. This seems introduced by #4028.

[vmamidi]

duplicate of #4903