10.2.x: Add log field fallbacks (#13018)

doc/admin-guide/logging/formatting.en.rst

@@ -314,8 +314,54 @@ prior to the log field's name, as so::
     Format = '%<{User-agent}cqh>'
 
 The above would insert the User Agent string from the client request headers
-into your log entry (or a blank string if no such header was present, or it did
-not contain a value).
+into your log entry (or ``-`` if no such header was present).
+
+Header fields can also be chained with a fallback operator, ``??``, when you want
+the log to use the first header that exists among n headers. For example::
+
+    Format = '%<{x-primary-id}cqh??{x-secondary-id}cqh??{x-tertiary-id}cqh>'
+
+|TS| evaluates the candidates from left to right and logs the first header that
+exists. If none of the headers exist, |TS| logs ``-`` by default. A header that
+exists but has an empty value is considered present, so |TS| logs the empty
+value instead of falling back. So in the example above, the value of
+x-primary-id of the client request is logged if it exists, otherwise the value
+of x-secondary-id is logged if it exists, otherwise ``-`` is logged if neither
+of the headers is present.
+
+The final log field in the chain can be a non-header log field whose value
+specifies the final fallback in the chain. This symbol is used only after all
+the previous header candidates in the chain are missing. For example::
+
+    Format = '%<{x-remote-ip}cqh??chi>'
+
+In this case, the value of the x-remote-ip HTTP header field is logged if that
+client request header exists. Otherwise, |TS| logs the value of ``chi``, the IP
+address of the client host.
+
+The final non-HTTP header log field must be the last term in the chain, so
+forms like ``%<chi??{x-id}cqh>`` are invalid.
+
+Alternatively, you can provide an explicit quoted default literal as the final
+term in the chain to use instead of the default ``-`` literal::
+
+    Format = '%<{x-primary-id}cqh??{x-secondary-id}cqh??"missing-id">'
+
+If none of the headers exist, |TS| logs the default literal instead,
+``missing-id`` in this case. The default literal must be quoted and must be the
+last term in the chain. Also, final non-header log fields and final default
+string literals cannot be used together. Thus forms like
+'%<{x-remote-ip}cqh??chi??"missing-id">' are invalid.
+
+
+Slices apply to each candidate in the fallback chain individually::
+
+    Format = '%<{x-primary-id}cqh[0:8]??{x-secondary-id}cqh[0:16]>'
+
+This is also true of non-header log fields. That is, if the final log field
+supports slicing, its own slice is preserved as usual::
+
+    Format = '%<{x-remote-ip}cqh??pqup[0:8]>'
 
 ===== ====================== ==================================================
 Field Source                 Description

include/proxy/logging/LogAccess.h

@@ -296,14 +296,16 @@ class LogAccess
   //
   // milestones access
   //
-  int  marshal_milestone(TSMilestonesType ms, char *buf);
-  int  marshal_milestone_fmt_sec(TSMilestonesType ms, char *buf);
-  int  marshal_milestone_fmt_squid(TSMilestonesType ms, char *buf);
-  int  marshal_milestone_fmt_netscape(TSMilestonesType ms, char *buf);
-  int  marshal_milestone_fmt_date(TSMilestonesType ms, char *buf);
-  int  marshal_milestone_fmt_time(TSMilestonesType ms, char *buf);
-  int  marshal_milestone_fmt_ms(TSMilestonesType ms, char *buf);
-  int  marshal_milestone_diff(TSMilestonesType ms1, TSMilestonesType ms2, char *buf);
+  int marshal_milestone(TSMilestonesType ms, char *buf);
+  int marshal_milestone_fmt_sec(TSMilestonesType ms, char *buf);
+  int marshal_milestone_fmt_squid(TSMilestonesType ms, char *buf);
+  int marshal_milestone_fmt_netscape(TSMilestonesType ms, char *buf);
+  int marshal_milestone_fmt_date(TSMilestonesType ms, char *buf);
+  int marshal_milestone_fmt_time(TSMilestonesType ms, char *buf);
+  int marshal_milestone_fmt_ms(TSMilestonesType ms, char *buf);
+  int marshal_milestone_diff(TSMilestonesType ms1, TSMilestonesType ms2, char *buf);
+
+  bool has_http_header_field(LogField::Container container, const char *field) const;
   void set_http_header_field(LogField::Container container, char *field, char *buf, int len);
   //
   // unmarshalling routines
@@ -398,8 +400,9 @@ class LogAccess
   char       *m_cache_lookup_url_canon_str        = nullptr;
   int         m_cache_lookup_url_canon_len        = 0;
 
-  void validate_unmapped_url();
-  void validate_unmapped_url_path();
+  HTTPHdr *header_for_container(LogField::Container container) const;
+  void     validate_unmapped_url();
+  void     validate_unmapped_url_path();
 
   void validate_lookup_url();
 };

include/proxy/logging/LogField.h

@@ -23,9 +23,12 @@
 
 #pragma once
 
+#include <memory>
+#include <optional>
 #include <string_view>
 #include <string>
 #include <variant>
+#include <vector>
 
 #include "tscore/ink_inet.h"
 #include "tscore/ink_platform.h"
@@ -126,13 +129,21 @@ class LogField
     N_AGGREGATES,
   };
 
+  struct HeaderField {
+    std::string name;
+    Container   container = NO_CONTAINER;
+    LogSlice    slice;
+  };
+
   LogField(const char *name, const char *symbol, Type type, MarshalFunc marshal, VarUnmarshalFuncSliceOnly unmarshal,
            SetFunc _setFunc = nullptr);
 
   LogField(const char *name, const char *symbol, Type type, MarshalFunc marshal, UnmarshalFuncWithMap unmarshal,
            const Ptr<LogFieldAliasMap> &map, SetFunc _setFunc = nullptr);
 
   LogField(const char *field, Container container);
+  LogField(const char *symbol, std::vector<HeaderField> header_fields, std::unique_ptr<LogField> fallback_field = nullptr,
+           std::optional<std::string> fallback_default = std::nullopt);
   LogField(const LogField &rhs);
   ~LogField();
 
@@ -188,6 +199,7 @@ class LogField
   static Container valid_container_name(char *name);
   static Aggregate valid_aggregate_name(char *name);
   static bool      fieldlist_contains_aggregates(const char *fieldlist);
+  static bool      isHeaderContainer(Container container);
   static bool      isContainerUpdateFieldSupported(Container container);
 
 private:
@@ -208,6 +220,17 @@ class LogField
   TSMilestonesType      milestone_from_m_name();
   int                   milestones_from_m_name(TSMilestonesType *m1, TSMilestonesType *m2);
 
+  std::vector<HeaderField>   m_fallback_header_fields;
+  std::unique_ptr<LogField>  m_fallback_field;
+  std::optional<std::string> m_fallback_default;
+  bool                       is_field_fallback() const;
+  int                        select_fallback_selector(LogAccess *lad) const;
+  unsigned                   marshal_fallback_header_field(LogAccess *lad, const HeaderField &field, char *buf) const;
+  unsigned                   marshal_fallback_default(char *buf) const;
+
+  static constexpr int FALLBACK_DEFAULT_SELECTOR = -1;
+  static constexpr int FALLBACK_FIELD_SELECTOR   = -2;
+
 public:
   LINK(LogField, link);
   LogSlice m_slice;

src/proxy/logging/CMakeLists.txt

@@ -26,6 +26,7 @@ add_library(
   LogFile.cc
   LogFilter.cc
   LogFormat.cc
+  LogFieldFallback.cc
   LogObject.cc
   LogUtils.cc
   RolledLogDeleter.cc
@@ -39,6 +40,11 @@ target_include_directories(logging PRIVATE ${SWOC_INCLUDE_DIR})
 target_link_libraries(logging PUBLIC ts::inkevent ts::inkutils ts::http ts::hdrs ts::tscore yaml-cpp::yaml-cpp)
 
 if(BUILD_TESTING)
+  add_executable(test_LogFieldFallback unit-tests/test_LogFieldFallback.cc)
+  target_include_directories(test_LogFieldFallback PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+  target_link_libraries(test_LogFieldFallback PRIVATE ts::logging Catch2::Catch2WithMain)
+  add_catch2_test(NAME test_LogFieldFallback COMMAND test_LogFieldFallback)
+
   add_executable(test_LogUtils LogUtils.cc unit-tests/test_LogUtils.cc)
   target_compile_definitions(test_LogUtils PRIVATE TEST_LOG_UTILS)
   target_link_libraries(test_LogUtils tscore ts::inkevent records Catch2::Catch2WithMain)

src/proxy/logging/LogAccess.cc

@@ -473,6 +473,54 @@ LogAccess::marshal_ip(char *dest, sockaddr const *ip)
   return INK_ALIGN_DEFAULT(len);
 }
 
+HTTPHdr *
+LogAccess::header_for_container(LogField::Container container) const
+{
+  switch (container) {
+  case LogField::CQH:
+  case LogField::ECQH:
+    return m_client_request;
+
+  case LogField::PSH:
+  case LogField::EPSH:
+    return m_proxy_response;
+
+  case LogField::PQH:
+  case LogField::EPQH:
+    return m_proxy_request;
+
+  case LogField::SSH:
+  case LogField::ESSH:
+    return m_server_response;
+
+  case LogField::CSSH:
+  case LogField::ECSSH:
+    return m_cache_response;
+
+  default:
+    return nullptr;
+  }
+}
+
+bool
+LogAccess::has_http_header_field(LogField::Container container, const char *field) const
+{
+  if (HTTPHdr const *header = header_for_container(container); header != nullptr) {
+    if (header->field_find(std::string_view{field}) != nullptr) {
+      return true;
+    }
+  }
+
+  if (container == LogField::SSH && strcmp(field, "Transfer-Encoding") == 0) {
+    const std::string &stored_te = m_http_sm->t_state.hdr_info.server_response_transfer_encoding;
+    if (!stored_te.empty()) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 inline int
 LogAccess::unmarshal_with_map(int64_t code, char *dest, int len, const Ptr<LogFieldAliasMap> &map, const char *msg)
 {
@@ -3225,33 +3273,7 @@ LogAccess::marshal_http_header_field(LogField::Container container, char *field,
   int      padded_len  = INK_MIN_ALIGN;
   int      actual_len  = 0;
   bool     valid_field = false;
-  HTTPHdr *header;
-
-  switch (container) {
-  case LogField::CQH:
-    header = m_client_request;
-    break;
-
-  case LogField::PSH:
-    header = m_proxy_response;
-    break;
-
-  case LogField::PQH:
-    header = m_proxy_request;
-    break;
-
-  case LogField::SSH:
-    header = m_server_response;
-    break;
-
-  case LogField::CSSH:
-    header = m_cache_response;
-    break;
-
-  default:
-    header = nullptr;
-    break;
-  }
+  HTTPHdr *header      = header_for_container(container);
 
   if (header) {
     MIMEField *fld = header->field_find(std::string_view{field});
@@ -3356,33 +3378,7 @@ LogAccess::marshal_http_header_field_escapify(LogField::Container container, cha
   int      padded_len = INK_MIN_ALIGN;
   int      actual_len = 0, new_len = 0;
   bool     valid_field = false;
-  HTTPHdr *header;
-
-  switch (container) {
-  case LogField::ECQH:
-    header = m_client_request;
-    break;
-
-  case LogField::EPSH:
-    header = m_proxy_response;
-    break;
-
-  case LogField::EPQH:
-    header = m_proxy_request;
-    break;
-
-  case LogField::ESSH:
-    header = m_server_response;
-    break;
-
-  case LogField::ECSSH:
-    header = m_cache_response;
-    break;
-
-  default:
-    header = nullptr;
-    break;
-  }
+  HTTPHdr *header      = header_for_container(container);
 
   if (header) {
     MIMEField *fld = header->field_find(std::string_view{field});