Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/Shellfish/IInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

public interface IInputSource
{
IEnumerable<string> GetInput();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're going for here.
My one basically using IObservable... IObservable is the mirror-image of IEnumerable so you end up with the same approach, just synchronous instead of async.

That said, while sync is much simpler, I'm not sure it's safe or correct 🤔 Will be fun to chat through it.

}
47 changes: 40 additions & 7 deletions source/Shellfish/ShellCommand.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ public class ShellCommand

List<IOutputTarget>? stdOutTargets;
List<IOutputTarget>? stdErrTargets;
IInputSource? stdInSource;

public ShellCommand(string executable)
{
Expand DownExpand Up@@ -138,18 +139,24 @@ public ShellCommand WithStdErrTarget(IOutputTarget target)
return this;
}

public ShellCommand WithStdInSource(IInputSource source)
{
stdInSource = source;
return this;
}

/// <summary>
/// Launches the process and synchronously waits for it to exit.
/// </summary>
public ShellCommandResult Execute(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedEvent = AttachProcessExitedManualResetEvent(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand DownExpand Up@@ -185,12 +192,12 @@ public ShellCommandResult Execute(CancellationToken cancellationToken = default)
public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
using var process = new Process();
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead);
ConfigureProcess(process, out var shouldBeginOutputRead, out var shouldBeginErrorRead, out var redirectingStdIn);

var exitedTask = AttachProcessExitedTask(process, cancellationToken);
process.Start();

BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead);
BeginIoStreams(process, shouldBeginOutputRead, shouldBeginErrorRead, redirectingStdIn);

try
{
Expand All@@ -216,7 +223,7 @@ public async Task<ShellCommandResult> ExecuteAsync(CancellationToken cancellatio
}

// sets standard flags on the Process that apply for both Execute and ExecuteAsync
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead)
void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool shouldBeginErrorRead, out bool redirectingStdIn)
{
process.StartInfo.FileName = executable;

Expand DownExpand Up@@ -264,7 +271,7 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
}
}

shouldBeginOutputRead = shouldBeginErrorRead = false;
shouldBeginOutputRead = shouldBeginErrorRead = redirectingStdIn = false;
if (stdOutTargets is { Count: > 0 })
{
process.StartInfo.RedirectStandardOutput = true;
Expand DownExpand Up@@ -292,15 +299,41 @@ void ConfigureProcess(Process process, out bool shouldBeginOutputRead, out bool
foreach (var target in targets) target.WriteLine(e.Data);
};
}

if (stdInSource is not null)
{
process.StartInfo.RedirectStandardInput = true;
redirectingStdIn = true;
}
}

// Common code for Execute and ExecuteAsync to handle stdin and stdout streaming
void BeginIoStreams(Process process,
bool shouldBeginOutputRead,
bool shouldBeginErrorRead)
bool shouldBeginErrorRead,
bool redirectingStdIn)
{
if (shouldBeginOutputRead) process.BeginOutputReadLine();
if (shouldBeginErrorRead) process.BeginErrorReadLine();

if (redirectingStdIn)
{
try
{
foreach (var val in stdInSource!.GetInput())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first iteration of my stdin one, I effectively wrote this; I just had an array of strings for stdin, and I wrote them all in a foreach loop exactly like this one.

I found though, that when I ran the test that asks for more than one line of input, it didn't work. What appeared to happen was that the first line of text would get sent to the first prompt, the second line of text would get dropped because the shell script hadn't yet asked for any input, and then shortly later when it did ask, it was too late.

I see you've brought that test across in this PR. Does it work for you? If so my hypothesis above must have been wrong in some way.

Anyway, while this should work well for simple programs, there are two things which give me pause

  1. We are single-threaded in the path of launching the process. If someone were to set up a stdin string for a program that wasn't expecting it, it might never read from stdin and our write would block forever. Our whole code-path would stall.
    1. Related to that: If our input enumerable takes a long time to enumerate, we'd also stall
    2. This approach works for small data sizes only because C# Streams have inbuilt buffering. If you exceed the buffer size, then writes will block until the other end starts to consume the data. Stall
  2. As I discovered, some programs exit when stdin is closed. octopus.server run is such a thing, and ross needed to pass in a dummy stdin stream that never closes, to stop it shutting itself down immediately. With an IEnumerable-based stream like this, how could we pull that off?

{
process.StandardInput.Write(val);
}
}
catch (OperationCanceledException)
{
// gracefully handle cancellation of the enumerator
}
finally
{
process.StandardInput.Close();
}
}
}

static async Task FinalWaitForExitAsync(Process process, CancellationToken cancellationToken)
Expand Down
17 changes: 17 additions & 0 deletions source/Shellfish/StringInputSource.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace Octopus.Shellfish;

class StringInputSource(string value) : IInputSource
{
public IEnumerable<string> GetInput() => [value];
}

public static class StringInputSourceExtensions
{
public static ShellCommand WithStdInSource(this ShellCommand shellCommand, string input)
{
shellCommand.WithStdInSource(new StringInputSource(input));
return shellCommand;
}
}
61 changes: 61 additions & 0 deletions source/Tests/Plumbing/TempScript.cs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Tests.Plumbing;

public static class TempScript
{
// Some interactions such as stdout or encoding codepages require things that don't work with an inline cmd /c or bash -c command
// This helper writes a script file into the temp directory so we can exercise more complex scenarios
public static Handle Create(string cmd, string sh)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".cmd");
File.WriteAllText(tempFile, cmd);
return new Handle(tempFile);
}
else
{
var tempFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".sh");
File.WriteAllText(tempFile, sh.Replace("\r\n", "\n"));
return new Handle(tempFile);
}
}

public class Handle(string scriptPath) : IDisposable
{
public string ScriptPath { get; } = scriptPath;

public void Dispose()
{
try
{
File.Delete(ScriptPath);
}
catch
{
// nothing to do if we can't delete the temp file
}
}

// Returns the host application which will run the script. Either cmd.exe or bash
public string GetHostExecutable()
{
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "bash";
}

// Returns the command line args to get the host application to run the script
// For cmd.exe, returns ["/c", ScriptPath] as it needs /c
// For bash, returns [ScriptPath] as it doesn't need any preamble
public string[] GetCommandArgs()
{
// when running cmd.exe we need /c to tell it to run the script; bash doesn't want any preamble for a script file
return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? ["/c", ScriptPath]
: [ScriptPath];

}
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdOutTarget
Octopus.Shellfish.ShellCommandExtensionMethods.WithStdErrTarget
Octopus.Shellfish.IInputSource.GetInput
Octopus.Shellfish.IOutputTarget.WriteLine
Octopus.Shellfish.ShellCommand.WithWorkingDirectory
Octopus.Shellfish.ShellCommand.WithArguments
Expand All@@ -11,10 +12,12 @@ Octopus.Shellfish.ShellCommand.WithCredentials
Octopus.Shellfish.ShellCommand.WithOutputEncoding
Octopus.Shellfish.ShellCommand.WithStdOutTarget
Octopus.Shellfish.ShellCommand.WithStdErrTarget
Octopus.Shellfish.ShellCommand.WithStdInSource
Octopus.Shellfish.ShellCommand.Execute
Octopus.Shellfish.ShellCommand.ExecuteAsync
Octopus.Shellfish.ShellCommandResult.ExitCode
Octopus.Shellfish.ShellExecutionException.Errors
Octopus.Shellfish.ShellExecutionException.Message
Octopus.Shellfish.ShellExecutor.ExecuteCommand
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.ShellExecutor.ExecuteCommandWithoutWaiting
Octopus.Shellfish.StringInputSourceExtensions.WithStdInSource
Loading