feat(profiler): native malloc/socket stack capture with profiler-hook frame skipping

ddprof-lib/src/main/cpp/codeCache.h

@@ -56,7 +56,7 @@ enum Mark {
  MARK_VM_RUNTIME = 1,
  MARK_INTERPRETER = 2,
  MARK_COMPILER_ENTRY = 3,
-MARK_ASYNC_PROFILER = 4, // async-profiler internals such as native hooks.
+MARK_JAVA_PROFILER = 4, // java-profiler internals such as native hooks.
  MARK_THREAD_ENTRY = 5, // Thread entry points (thread_native_entry, JavaThread::, etc.)
 };
 
@@ -246,7 +246,7 @@ class CodeCache {
  * Mark symbols matching the predicate with the given mark value.
  *
  * This is called during profiler initialization to mark JVM internal functions
- * (MARK_VM_RUNTIME, MARK_INTERPRETER, MARK_COMPILER_ENTRY, MARK_ASYNC_PROFILER).
+ * (MARK_VM_RUNTIME, MARK_INTERPRETER, MARK_COMPILER_ENTRY, MARK_JAVA_PROFILER).
  */
  template <typename NamePredicate>
  inline void mark(NamePredicate predicate, char value) {

ddprof-lib/src/main/cpp/counters.h

@@ -119,6 +119,8 @@
  X(JVMTI_STACKS_INIT_OK, "jvmti_stacks_init_ok") \
  X(JVMTI_STACKS_INIT_FAILED, "jvmti_stacks_init_failed") \
  X(JVMTI_STACKS_REQUESTED, "jvmti_stacks_requested") \
+ X(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND, "native_trace_hook_prefix_not_found") \
+ X(NATIVE_HOOK_MARK_RESOLVE_FAILED, "native_hook_mark_resolve_failed") \
  X(JVMTI_STACKS_FAILED_WRONG_PHASE, "jvmti_stacks_failed_wrong_phase") \
  X(JVMTI_STACKS_FAILED_OTHER, "jvmti_stacks_failed_other") \
  /* Delegated stacks dropped at slot-lock. Rec-lock drops from all recording \

ddprof-lib/src/main/cpp/event.h

@@ -34,6 +34,7 @@ enum EventType {
  EXECUTION_SAMPLE,
  WALL_CLOCK_SAMPLE,
  MALLOC_SAMPLE,
+ SOCKET_SAMPLE,
  INSTRUMENTED_METHOD,
  METHOD_TRACE,
  ALLOC_SAMPLE,

ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp

@@ -126,6 +126,8 @@ inline EventType eventTypeFromBCI(jint bci_type) {
  return PARK_SAMPLE;
  case BCI_NATIVE_MALLOC:
  return MALLOC_SAMPLE;
+ case BCI_NATIVE_SOCKET:
+ return SOCKET_SAMPLE;
  default:
  // For unknown or invalid BCI types, default to EXECUTION_SAMPLE
  // This maintains backward compatibility and prevents undefined behavior
@@ -305,8 +307,14 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
  uintptr_t saved_anchor_fp = 0;
  bool anchor_recovery_used = false;
 
+ // Set once the MARK_JAVA_PROFILER hook boundary is found for a
+ // malloc/socket sample — mirrors skip_hook_prefix/skipping in
+ // Profiler::convertNativeTrace so the same "boundary never found"
+ // condition is observable from walkVM.
+ bool hook_boundary_found = false;
+
  // Show extended frame types and stub frames for execution-type events
- bool details = event_type <= MALLOC_SAMPLE || features.mixed;
+ bool details = event_type <= SOCKET_SAMPLE || features.mixed;
 
  if (details && vm_thread != NULL && VMThread::isJavaThread(vm_thread)) {
  anchor = vm_thread->anchor();
@@ -681,18 +689,16 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
  } else {
  // Resolve native frame (may use remote symbolication if enabled)
  Profiler::NativeFrameResolution resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)pc, lock_index);
- if (resolution.is_marked) {
- // This is a marked C++ interpreter frame, terminate scan
- break;
- }
- const char* method_name = resolution.method_name;
- int frame_bci = resolution.bci;
- char mark;
- if (frame_bci != BCI_NATIVE_FRAME_REMOTE && method_name != NULL && (mark = NativeFunc::read_mark(method_name)) != 0) {
- if (mark == MARK_ASYNC_PROFILER && event_type == MALLOC_SAMPLE) {
- // Skip all internal frames above malloc_hook functions, leave the hook itself
+ if (resolution.is_marked()) {
+ if (resolution.mark == MARK_JAVA_PROFILER &&
+ isHookPrefixedSample(event_type)) {
+ // Discard frames captured above the malloc/socket hook boundary,
+ // excluding the hook's own frame, and resume from the real
+ // caller above it — mirrors the FP/DWARF skip-prefix logic in
+ // Profiler::convertNativeTrace.
+ hook_boundary_found = true;
  depth = 0;
- } else if (mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) {
+ } else if (resolution.mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) {
  // Insert current compile task as a pseudo Java frame
  VMMethod* method = vm_thread->compiledMethod();
  if (method != nullptr) {
@@ -701,13 +707,19 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
  fillFrame(frames[depth++], FRAME_JIT_COMPILED, 0, method_id, method);
  }
  }
- } else if (mark == MARK_THREAD_ENTRY) {
+ } else if (resolution.mark == MARK_THREAD_ENTRY) {
  // Thread entry point detected via pre-computed mark - this is the root frame
- // No need for expensive symbol resolution, just stop unwinding
  Counters::increment(THREAD_ENTRY_MARK_DETECTIONS);
  break;
+ } else {
+ // Other marks (VM runtime / interpreter) terminate the scan.
+ break;
  }
- } else if (method_name == NULL && details && !anchor_recovery_used
+ goto dwarf_unwind;
+ }
+ const char* method_name = resolution.method_name;
+ int frame_bci = resolution.bci;
+ if (method_name == NULL && details && !anchor_recovery_used
  && profiler->findLibraryByAddress(pc) == NULL) {
  // Try anchor recovery — prefer live anchor, fall back to saved data
  anchor_recovery_used = true;
@@ -945,6 +957,12 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
  Counters::increment(WALKVM_DEPTH_ZERO);
  }
 
+ if (isHookPrefixedSample(event_type) && !hook_boundary_found) {
+ // The malloc/socket hook boundary was never found in this walk;
+ // mirrors Profiler::convertNativeTrace's NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND.
+ Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+ }
+
  if (truncated) {
  if (depth > max_depth) {
  *truncated = true;
@@ -1198,7 +1216,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) {
  int java_frames = 0;
  if (features.mixed) {
  java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated);
- } else if (request.event_type == BCI_NATIVE_MALLOC || request.event_type == BCI_NATIVE_SOCKET) {
+ } else if (isHookPrefixedSample(request.event_type)) {
  if (cstack >= CSTACK_VM) {
  java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated);
  } else {

ddprof-lib/src/main/cpp/jfrMetadata.cpp

@@ -303,8 +303,8 @@ void JfrMetadata::initialize(
  << field("name", T_STRING, "Name")
  << field("count", T_LONG, "Count"))
 
- << (type("profiler.Malloc", T_MALLOC, "malloc")
- << category("Java Virtual Machine", "Native Memory")
+ << (type("datadog.NativeMemoryAllocation", T_MALLOC, "Native Memory Allocation")
+ << category("Datadog", "Profiling")
  << field("startTime", T_LONG, "Start Time", F_TIME_TICKS)
  << field("eventThread", T_THREAD, "Event Thread", F_CPOOL)
  << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL)

ddprof-lib/src/main/cpp/mallocTracer.cpp

@@ -231,7 +231,7 @@ bool MallocHooker::initialize() {
  || strcmp(s, "posix_memalign_hook") == 0
  || strcmp(s, "aligned_alloc_hook") == 0;
  },
-MARK_ASYNC_PROFILER);
+MARK_JAVA_PROFILER);
 
  splitmix64_seed(TSC::ticks());
  _initialized = true;

ddprof-lib/src/main/cpp/nativeSocketSampler.cpp

@@ -7,8 +7,11 @@
 
 #if defined(__linux__)
 
+#include "codeCache.h"
 #include "common.h"
+#include "counters.h"
 #include "flightRecorder.h"
+#include "libraries.h"
 #include "libraryPatcher.h"
 #include "log.h"
 #include "os.h"
@@ -27,6 +30,30 @@
 static thread_local PoissonSampler _send_sampler;
 static thread_local PoissonSampler _recv_sampler;
 
+// Marks the hook wrapper's own symbol as MARK_JAVA_PROFILER so native call-stack
+// unwinding (Profiler::convertNativeTrace) can recognize the boundary between
+// profiler-internal frames and the real caller, mirroring MallocHooker::initialize().
+// Resolved by address (not by symbol-name predicate) because these are mangled
+// C++ static member functions, unlike malloc_hook's extern "C" free functions.
+// Returns false if the symbol could not be resolved/marked, in which case the
+// hook boundary is never recognized and every socket sample's native stack
+// comes back empty (see NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND).
+static bool markJavaProfilerHook(void* fn_addr) {
+ CodeCache* lib = Libraries::instance()->findLibraryByAddress(fn_addr);
+ if (lib == nullptr) {
+ Counters::increment(NATIVE_HOOK_MARK_RESOLVE_FAILED);
+ return false;
+ }
+ const char* name = nullptr;
+ lib->binarySearch(fn_addr, &name);
+ if (name == nullptr) {
+ Counters::increment(NATIVE_HOOK_MARK_RESOLVE_FAILED);
+ return false;
+ }
+ NativeFunc::set_mark(name, MARK_JAVA_PROFILER);
+ return true;
+}
+
 // Debug-only hook-fire counters, paired with TEST_LOG (common.h). Gated at
 // compile time to keep release hot paths free of cross-thread atomic writes.
 #ifdef DEBUG
@@ -386,6 +413,18 @@ Error NativeSocketSampler::start(Arguments &args) {
  TEST_LOG("NativeSocketSampler::start interval_ticks=%ld tsc_freq=%llu",
  init_interval, (unsigned long long)TSC::frequency());
 #endif
+ bool hooks_marked = markJavaProfilerHook((void*)&NativeSocketSampler::send_hook);
+ hooks_marked &= markJavaProfilerHook((void*)&NativeSocketSampler::recv_hook);
+ hooks_marked &= markJavaProfilerHook((void*)&NativeSocketSampler::write_hook);
+ hooks_marked &= markJavaProfilerHook((void*)&NativeSocketSampler::read_hook);
+ if (!hooks_marked) {
+ // Not fatal: hooks are still installed and sampling still works, but
+ // native stacks for socket samples will come back empty because the
+ // hook boundary frame can't be recognized during unwinding.
+ Log::warn("NativeSocketSampler: failed to mark one or more hook symbols; "
+ "native call stacks for socket samples may be empty");
+ }
+
  if (!LibraryPatcher::patch_socket_functions()) {
  return Error("failed to install native socket hooks (dlsym returned NULL)");
  }

ddprof-lib/src/main/cpp/profiler.cpp

@@ -302,6 +302,7 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
  if (_cstack == CSTACK_NO ||
  (event_type == BCI_ALLOC || event_type == BCI_ALLOC_OUTSIDE_TLAB) ||
  (event_type != BCI_CPU && event_type != BCI_WALL &&
+ !isHookPrefixedSample(event_type) &&
  _cstack == CSTACK_DEFAULT)) {
  return 0;
  }
@@ -326,7 +327,9 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
  java_ctx, truncated);
  }
 
- return convertNativeTrace(native_frames, callchain, frames, lock_index);
+ bool skip_hook_prefix = isHookPrefixedSample(event_type);
+ return convertNativeTrace(native_frames, callchain, frames, lock_index,
+ skip_hook_prefix);
 }
 
 /**
@@ -383,24 +386,27 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
  char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0;
 
  if (mark != 0) {
- return {nullptr, BCI_NATIVE_FRAME, true}; // Marked - stop processing
+ return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, mark); // Marked - caller dispatches on mark
  }
 
  // Pack remote symbolication data using utility struct
  uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase();
  uint32_t lib_index = (uint32_t)lib->libIndex();
  unsigned long packed = RemoteFramePacker::pack(pc_offset, mark, lib_index);
 
- return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false);
+ return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE);
  }
 
  // Traditional symbol resolution
  const char *method_name = nullptr;
  if (lib != nullptr) {
  lib->binarySearch((void*)pc, &method_name);
  }
- if (method_name != nullptr && NativeFunc::is_marked(method_name)) {
- return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, true);
+ if (method_name != nullptr) {
+ char mark = NativeFunc::read_mark(method_name);
+ if (mark != 0) {
+ return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, mark);
+ }
  }
 
  // No symbol but known library: pack for library-relative identification.
@@ -410,10 +416,10 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
  uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase();
  uint32_t lib_index = (uint32_t)lib->libIndex();
  unsigned long packed = RemoteFramePacker::pack(pc_offset, 0, lib_index);
- return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false);
+ return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE);
  }
 
- return NativeFrameResolution(method_name, BCI_NATIVE_FRAME, false);
+ return NativeFrameResolution(method_name, BCI_NATIVE_FRAME);
 }
 
 /**
@@ -426,9 +432,16 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
  * marked frames (JVM internals) that should terminate the stack walk.
  */
 int Profiler::convertNativeTrace(int native_frames, const void **callchain,
- ASGCT_CallFrame *frames, int lock_index) {
+ ASGCT_CallFrame *frames, int lock_index,
+ bool skip_hook_prefix) {
  int depth = 0;
  void* prev_identifier = NULL; // Can be jmethodID or frame pointer for remote
+ // skip_hook_prefix: the walk started inside profiler-internal code (e.g. the
+ // malloc/socket hook call chain), not at an interrupted user PC. Discard frames
+ // until the hook wrapper's own MARK_JAVA_PROFILER-marked frame is reached, then
+ // resume normally from the real caller. Other mark kinds still terminate the
+ // scan immediately, same as the non-skipping case.
+ bool skipping = skip_hook_prefix;
 
  for (int i = 0; i < native_frames; i++) {
  uintptr_t pc = (uintptr_t)callchain[i];
@@ -444,9 +457,20 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
  char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0;
 
  if (mark != 0) {
+ if (skip_hook_prefix && mark == MARK_JAVA_PROFILER) {
+ depth = 0;
+ skipping = false;
+ continue;
+ }
+ if (skipping) {
+ // A non-MARK_JAVA_PROFILER mark terminated the scan before the
+ // hook boundary was ever found; the sample has no native stack.
+ Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+ }
  // Terminate scan at marked frame
  return depth;
  }
+ if (skipping) continue;
 
  // Populate remote frame inline - no allocation needed!
  // Pass the mark we already retrieved to avoid duplicate binarySearch
@@ -464,10 +488,24 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
 
  // Fallback: Traditional symbol resolution
  const char *method_name = findNativeMethod((void*)pc);
- if (method_name != nullptr && NativeFunc::is_marked(method_name)) {
- // Terminate scan at marked frame
- return depth;
+ if (method_name != nullptr) {
+ char mark = NativeFunc::read_mark(method_name);
+ if (mark != 0) {
+ if (skip_hook_prefix && mark == MARK_JAVA_PROFILER) {
+ depth = 0;
+ skipping = false;
+ continue;
+ }
+ if (skipping) {
+ // A non-MARK_JAVA_PROFILER mark terminated the scan before the
+ // hook boundary was ever found; the sample has no native stack.
+ Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+ }
+ // Terminate scan at marked frame
+ return depth;
+ }
  }
+ if (skipping) continue;
 
  // Store standard frame
  jmethodID current_method = (jmethodID)method_name;
@@ -481,6 +519,11 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
  }
  }
 
+ if (skipping) {
+ // The hook-boundary (MARK_JAVA_PROFILER) frame was never found in the
+ // callchain; every frame was discarded and the sample has no native stack.
+ Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+ }
  return depth;
 }