Extraction patterns

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
  readonly FileInputFeature _fileInputFeature;
  readonly PropertiesFeature _properties;
  readonly ConnectionFeature _connection;
- string _filter;
+ string _filter, _pattern;
  bool _json;
 
  public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
  _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
  _properties = Enable<PropertiesFeature>();
 
+ Options.Add("x=|extract=",
+ "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+ v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
  Options.Add("json",
  "Read the events as JSON (the default assumes plain text)",
  v => _json = true);
 
  Options.Add("f=|filter=",
  "Filter expression to select a subset of events",
  v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+
  _connection = Enable<ConnectionFeature>();
  }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
  var reader = _json ?
  (ILogEventReader)new ClefLogEventReader(input) :
- new PlainTextLogEventReader(input);
+ new PlainTextLogEventReader(input, _pattern);
 
  using (reader as IDisposable)
  {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+ static class ExtractionPatternInterpreter
+ {
+ public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+ {
+ new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+ });
+
+ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+ {
+ if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+ var patternElements = new PatternElement[pattern.Elements.Count];
+ for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+ {
+ var element = pattern.Elements[i];
+ switch (element)
+ {
+ case LiteralTextPatternExpression text:
+ patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is NonGreedyContentExpression ngc:
+ patternElements[i] = new PatternElement(
+ Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+ capture.Name);
+ break;
+ case CapturePatternExpression capture
+ when capture.Content is MatchTypeContentExpression mtc:
+ patternElements[i] = new PatternElement(
+ mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+ capture.Name);
+ break;
+ default:
+ throw new InvalidOperationException($"Element `{element}` not recognized.");
+ }
+ }
+
+ return new NameValueExtractor(patternElements);
+ }
+
+ // What we need to do here is:
+ // - for each parsed token
+ // - if it's literal text, map it an anonymous PatternElement with
+ // BuiltInPatterns.LiteralText()
+ // - otherwise, if it specifies no format, it's a named element with
+ // the BuiltInPatterns.Token parser
+ // - if it does specify a format, look up the parser based on the name, except
+ // - if the format is `$` it is BuiltInPatterns.SingleLineContent
+ // - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+ // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+ // parser that follows it
+ }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+ [AttributeUsage(AttributeTargets.Property)]
+ class MatcherAttribute : Attribute
+ {
+ public string Name { get; }
+
+ public MatcherAttribute(string name)
+ {
+ Name = name ?? throw new ArgumentNullException(nameof(name));
+ }
+ }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+ // ReSharper disable UnusedMember.Global
+ static class Matchers
+ {
+ [Matcher("ident")]
+ public static TextParser<object> Identifier { get; } =
+ IdentifierEx.CStyle
+ .Select(span => (object) span);
+
+ [Matcher("nat")]
+ public static TextParser<object> Natural { get; } =
+ Numerics.NaturalUInt64
+ .Select(span => (object) span);
+
+ [Matcher("int")]
+ public static TextParser<object> Integer { get; } =
+ Numerics.IntegerInt64
+ .Select(span => (object) span);
+
+ [Matcher("dec")]
+ public static TextParser<object> Decimal { get; } =
+ NumericsEx.Decimal
+ .Select(span => (object) span);
+
+ [Matcher("alpha")]
+ public static TextParser<object> Alphabetical { get; } =
+ Span.WithAll(char.IsLetter)
+ .Select(span => (object) span);
+
+ [Matcher("alphanum")]
+ public static TextParser<object> Alphanumeric { get; } =
+ Span.WithAll(char.IsLetterOrDigit)
+ .Select(span => (object) span);
+
+ [Matcher("token")]
+ public static TextParser<object> Token { get; } =
+ SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+ [Matcher("iso8601dt")]
+ // A date and time are required by this pattern, though not necessarily by the spec.
+ public static TextParser<object> Iso8601DateTime { get; } =
+ DateTimesEx.Iso8601DateTime
+ .Select(span => (object) span);
+
+ public static TextParser<object> SerilogFileTimestamp { get; } =
+ Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+ .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+ [Matcher("timestamp")]
+ public static TextParser<object> Timestamp { get; } =
+ Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+ // Unclear whether we need to name this
+ public static TextParser<object> MultiLineMessage { get; } =
+ SpanEx.MatchedBy(
+ Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+ .IgnoreThen(Character.AnyChar.Many()))
+ .Select(span => (object)span);
+
+ // Equivalent to :* at end-of-pattern
+ public static TextParser<object> MultiLineContent { get; } =
+ Span.WithAll(ch => true)
+ .Select(span => (object)span);
+
+ [Matcher("line")]
+ public static TextParser<object> SingleLineContent { get; } =
+ from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+ from _ in NewLine.OptionalOrDefault()
+ select (object) content;
+
+ [Matcher("n")]
+ public static TextParser<object> NewLine { get; } =
+ Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+ .Select(span => (object)span);
+
+ [Matcher("t")]
+ public static TextParser<object> Tab { get; } =
+ Span.EqualTo("\t")
+ .Select(span => (object)span);
+
+ static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+ from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+ let attr = pi.GetCustomAttribute<MatcherAttribute>()
+ where attr != null
+ select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+ public static TextParser<object> GetByType(string type)
+ {
+ if (type == null) throw new ArgumentNullException(nameof(type));
+ return ByType[type];
+ }
+
+ public static TextParser<object> LiteralText(string literalText)
+ {
+ return Span.EqualTo(literalText).Select(span => (object) span);
+ }
+
+ public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+ {
+ if (following.Length == 0)
+ return SpanEx.MatchedBy(Character.AnyChar.Many())
+ .Select(span => span.Length > 0 ? (object) span : null);
+
+ var rest = following[0].Parser;
+ for (var i = 1; i < following.Length; ++i)
+ {
+ rest = rest.IgnoreThen(following[i].Parser);
+ }
+
+ return i =>
+ {
+ var remainder = i;
+ var attempt = rest(remainder);
+ while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+ {
+ remainder = remainder.ConsumeChar().Remainder;
+ attempt = rest(remainder);
+ }
+
+ var span = i.Until(remainder);
+ return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+ };
+ }
+ }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
- class Pattern
+ class NameValueExtractor
  {
  readonly PatternElement[] _elements;
 
- public Pattern(IEnumerable<PatternElement> elements)
+ public NameValueExtractor(IEnumerable<PatternElement> elements)
  {
  _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
  if (_elements.Length == 0)
- throw new ArgumentException("A match pattern must contain at least one element."); 
+ throw new ArgumentException("An extraction pattern must contain at least one element."); 
  }
 
- public TextParser<object> FrameStart => _elements[0].Parser;
+ public TextParser<object> StartMarker => _elements[0].Parser;
 
- public (IDictionary<string, object>, string) Match(string frame)
+ public (IDictionary<string, object>, string) ExtractValues(string plainText)
  {
- var input = new TextSpan(frame);
+ var input = new TextSpan(plainText);
  var result = new Dictionary<string, object>();
 
  var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
  if (!element.IsIgnored)
  {
- if (match.Value != null || !element.IsOptional)
- result.Add(element.Name, match.Value); 
+ result.Add(element.Name, match.Value); 
  }
  }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
  class PatternElement
  {
- public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+ public PatternElement(TextParser<object> parser, string name = null)
  {
  Parser = parser ?? throw new ArgumentNullException(nameof(parser));
  Name = name;
- IsOptional = isOptional;
  }
 
  public TextParser<object> Parser { get; }
  public string Name { get; }
- public bool IsOptional { get; }
  public bool IsIgnored => Name == null;
  }
 }