More accurate argv parsing/serialization on Windows

[castholm]

The first commit adds ArgIteratorWindows, which faithfully replicates the quoting and escaping behavior observed in CommandLineToArgvW and should make Zig applications play better with processes that abuse these quirks. ArgIterator is updated to use this implementation on Windows.

The new implementation used ArgIteratorGeneral as a base but the new rules were written by me, from scratch, by following public documentation and manually observing the behavior of CommandLineToArgvW on Windows 10; they weren't ported from a different project or based on disassembly.

I left ArgIteratorGeneral as is, but since it is now only used for response files it might make sense to rename it ArgIteratorResponseFile and remove the Windows-specific wide string init overload.

The second commit updates the argv-to-command-line serialization in ChildProcess on Windows to follow these rules properly (including the special quoting rules for the first argument), which fixes a bug where empty arguments weren't quoted and thus ignored by child processes.

The PR includes test cases for both parsing and serialization. If you would like to independently verify that the argv parsing matches CommandLineToArgvW you can paste the test cases from test "ArgIteratorWindows" into this program and run it (only works on Windows for obvious reasons):

//! test_argv.zigconststd=@import("std");
vararena: std.heap.ArenaAllocator=undefined;
varallocator: std.mem.Allocator=undefined;
pubfnmain() !void {
arena=std.heap.ArenaAllocator.init(std.heap.page_allocator);
deferarena.deinit();
allocator=arena.allocator();
constt=testCommandLine;
// PASTE TEST CASES HERE
}
fntestCommandLine(cmd_line: []constu8, expected_args: []const []constu8) !void {
constcmd_line_w=trystd.unicode.utf8ToUtf16LeWithNull(allocator, cmd_line);
constcmd_line_esc=trystd.Uri.escapeStringWithFn(allocator, cmd_line, keepUnescaped);
std.debug.print("cmd_line: ({s})\n", .{cmd_line_esc});
std.debug.print("exp:", .{});
for (expected_args) |arg| {
constarg_esc=trystd.Uri.escapeStringWithFn(allocator, arg, keepUnescaped);
std.debug.print(" ({s})", .{arg_esc});
}
std.debug.print("\n", .{});
std.debug.print("win:", .{});
constwin_args=trywindowsParseCommandLine(cmd_line_w);
for (win_args) |arg| {
constarg_esc=trystd.Uri.escapeStringWithFn(allocator, arg, keepUnescaped);
std.debug.print(" ({s})", .{arg_esc});
}
std.debug.print("\n", .{});
std.debug.print("zig:", .{});
varzig_args=trystd.process.ArgIteratorWindows.init(allocator, cmd_line_w);
while (zig_args.next()) |arg| {
constarg_esc=trystd.Uri.escapeStringWithFn(allocator, arg, keepUnescaped);
std.debug.print(" ({s})", .{arg_esc});
}
std.debug.print("\n", .{});
std.debug.print("----\n", .{});
}
fnkeepUnescaped(c: u8) bool {
returnstd.ascii.isPrint(c) andc!=' 'andc!='('andc!=')';
}
fnwindowsParseCommandLine(cmd_line_w: [:0]constu16) ![][:0]u8 {
varargc: c_int=undefined;
constargv_w=CommandLineToArgvW(cmd_line_w, &argc) orelsereturnerror.Unexpected;
deferstd.os.windows.LocalFree(@ptrCast(argv_w));
constargs=tryallocator.alloc([:0]u8, @intCast(argc));
for (argv_w, 0..@intCast(argc)) |arg_w, i| {
args[i] =trystd.unicode.utf16leToUtf8AllocZ(allocator, std.mem.span(arg_w));
}
returnargs;
}
extern"shell32"fnCommandLineToArgvW(lpCmdLine: std.os.windows.LPCWSTR, pNumArgs: *c_int) ?[*]std.os.windows.LPCWSTR;

zig run ./test_argv.zig

This program will print out URI-escaped lines that help you compare the output of the Windows and Zig implementations to the expected result.

cmd_line: ("C:\Program%20Files\zig\zig.exe"%20run%20.\src\main.zig)
exp: (C:\Program%20Files\zig\zig.exe) (run) (.\src\main.zig)
win: (C:\Program%20Files\zig\zig.exe) (run) (.\src\main.zig)
zig: (C:\Program%20Files\zig\zig.exe) (run) (.\src\main.zig)
----
cmd_line: (foo.exe%20bar)
exp: (foo.exe) (bar)
win: (foo.exe) (bar)
zig: (foo.exe) (bar)
----

Happy to answer any questions or concerns you may have.

[squeek502]

Awesome work, love the methodology used!

I modified your test_argv.zig to do some fuzzing-like generation of ASCII-only command lines and then had it check that the result always matches CommandLineToArgvW. For context, it generates random command lines that look something like this (represented here as a Zig string literal):

"\t \"\t\\\"\t\"\\ \" \\\\\"5c\\\t\t\t\" \\ \\;\x0b\t\t"

Test code

conststd=@import("std");
constAllocator=std.mem.Allocator;
pubfnmain() !void {
vargpa=std.heap.GeneralPurposeAllocator(.{}){};
deferstd.debug.assert(gpa.deinit() ==.ok);
constallocator=gpa.allocator();
varrandom=std.rand.DefaultPrng.init(0);
constrand=random.random();
vari: usize=0;
while (true) : (i+=1) {
constrand_cmd_line=tryrandomCommandLine(allocator, rand);
deferallocator.free(rand_cmd_line);
// TEMPORARY to avoid known difference for 0-length command linesif (rand_cmd_line.len==0orrand_cmd_line[0] =='\x00') continue;
trytestCommandLine(allocator, rand_cmd_line);
if (i%1000==0) {
std.debug.print("{}\r", .{i});
}
}
}
fntestCommandLine(allocator: Allocator, cmd_line: []constu8) !void {
constcmd_line_w=trystd.unicode.utf8ToUtf16LeWithNull(allocator, cmd_line);
deferallocator.free(cmd_line_w);
constwin_args=trywindowsParseCommandLine(allocator, cmd_line_w);
defer {
for (win_args) |win_arg| {
allocator.free(win_arg);
}
allocator.free(win_args);
}
varzig_args=trystd.process.ArgIteratorWindows.init(allocator, cmd_line_w);
deferzig_args.deinit();
varzig_args_list=std.ArrayList([]constu8).init(allocator);
deferzig_args_list.deinit();
while (zig_args.next()) |zig_arg| {
tryzig_args_list.append(zig_arg);
}
if (getDiffIndex(win_args, zig_args_list.items)) |diff_index| {
std.debug.print("cmd_line: \"{}\"\n", .{std.zig.fmtEscapes(cmd_line)});
std.debug.print("first difference at index {}\n", .{diff_index});
if (diff_index>=win_args.len)
std.debug.print("expected: null\n", .{})
elsestd.debug.print("expected: \"{}\"\n", .{std.zig.fmtEscapes(win_args[diff_index])});
if (diff_index>=zig_args_list.items.len)
std.debug.print("actual: null\n", .{})
elsestd.debug.print("actual: \"{}\"\n", .{std.zig.fmtEscapes(zig_args_list.items[diff_index])});
std.debug.print("win:", .{});
for (win_args) |arg| {
std.debug.print(" \"{}\"", .{std.zig.fmtEscapes(arg)});
}
std.debug.print("\n", .{});
std.debug.print("zig:", .{});
for (zig_args_list.items) |arg| {
std.debug.print(" \"{}\"", .{std.zig.fmtEscapes(arg)});
}
std.debug.print("\n", .{});
returnerror.ParsedArgsNotEqual;
}
}
fngetDiffIndex(a: []const []constu8, b: []const []constu8) ?usize {
if (a.len!=b.len) return@min(a.len, b.len);
for (a, b, 0..) |a_str, b_str, i| {
if (!std.mem.eql(u8, a_str, b_str)) returni;
}
returnnull;
}
fnwindowsParseCommandLine(allocator: Allocator, cmd_line_w: [:0]constu16) ![][:0]u8 {
varargc: c_int=undefined;
constargv_w=CommandLineToArgvW(cmd_line_w, &argc) orelsereturnerror.Unexpected;
deferstd.os.windows.LocalFree(@ptrCast(argv_w));
constargs=tryallocator.alloc([:0]u8, @intCast(argc));
for (argv_w, 0..@intCast(argc)) |arg_w, i| {
args[i] =trystd.unicode.utf16leToUtf8AllocZ(allocator, std.mem.span(arg_w));
}
returnargs;
}
fnrandomCommandLine(allocator: Allocator, rand: std.rand.Random) ![]constu8 {
constChoice=enum {
backslash,
quote,
space,
tab,
control,
printable,
};
constchoices=rand.uintAtMostBiased(u16, 256);
varbuf=trystd.ArrayList(u8).initCapacity(allocator, choices);
errdeferbuf.deinit();
for (0..choices) |_| {
constchoice=rand.enumValue(Choice);
constbyte=switch (choice) {
.backslash=>'\\',
.quote=>'"',
.space=>' ',
.tab=>'\t',
.control=>switch (rand.uintAtMostBiased(u8, 0x21)) {
0x21=>'\x7F',
else=>|b|b,
},
.printable=>'!'+rand.uintAtMostBiased(u8, '~'-'!'),
};
trybuf.append(byte);
}
returnbuf.toOwnedSlice();
}
extern"shell32"fnCommandLineToArgvW(lpCmdLine: std.os.windows.LPCWSTR, pNumArgs: *c_int) ?[*]std.os.windows.LPCWSTR;

The only difference found in 16,000,000+ iterations so far is around passing a 0-length string to CommandLineToArgvW (either "" or "\x00..."). In that case, CommandLineToArgvW seemingly returns what std.fs.selfExePath would return:

cmd_line: ""
first difference at index 0
expected: "C:\\Users\\Ryan\\AppData\\Local\\zig\\o\\7678c5300c1aba434f81e13eb0b7e531\\test_argv.exe"
actual: null
win: "C:\\Users\\Ryan\\AppData\\Local\\zig\\o\\7678c5300c1aba434f81e13eb0b7e531\\test_argv.exe"
zig:

Not sure if/how that should be handled on the Zig side (I currently specifically skip it in the test code--comment that line out if you want to hit that case yourself).

[castholm]

FWIW I made sure to document that the only difference in behavior between ArgIteratorWindows and CommandLineToArgvW is that an empty command-line string will yield no arguments instead of the name of the current executable.

It's also worth taking into consideration how CreateProcessW works:

• lpApplicationName is the executable name and lpCommandLine is the command-line string.
• If lpCommandLine is null (or the empty string, as I found out by testing), the value of lpApplicationName is used as the command-line string.
• If lpApplicationName is null, the first argument in lpCommandLine is used as the executable name.

https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw

This means that the child process' command-line string retrieved via GetCommandLineW will never actually be empty in practice, so the edge case is probably not worth handling (unless there's a different way to spawn a process on Windows with drastically different behavior that I'm unaware of).

You can verify the behavior of lpApplicationName/lpCommandLine and the resulting command-line string using these two example programs:

Details

//! echo_cmd_line.zigconststd=@import("std");
pubfnmain() !void {
constcmd_line_w=std.os.windows.kernel32.GetCommandLineW();
conststdout=std.io.getStdOut().writer();
trystdout.print("{s}", .{std.unicode.fmtUtf16le(std.mem.span(cmd_line_w))});
}

zig build-exe ./echo_cmd_line.zig

//! test_cmd_line.zigconststd=@import("std");
constwindows=std.os.windows;
pubfnmain() !void {
vargpa=std.heap.GeneralPurposeAllocator(.{}){};
deferstd.debug.assert(gpa.deinit() ==.ok);
constallocator=gpa.allocator();
constecho_app_name=get_echo_app_name: {
varargc: c_int=undefined;
constargv_w=CommandLineToArgvW(windows.kernel32.GetCommandLineW(), &argc) orelsereturnerror.Unexpected;
deferwindows.LocalFree(@ptrCast(argv_w));
if (argc<2) returnerror.MissingEchoAppNameArg;
break :get_echo_app_nametrystd.unicode.utf16leToUtf8Alloc(allocator, std.mem.span(argv_w[1]));
};
deferallocator.free(echo_app_name);
constquoted_echo_app_name=trystd.fmt.allocPrint(allocator, "\"{s}\"", .{echo_app_name});
deferallocator.free(quoted_echo_app_name);
inlinefor (.{
.{ echo_app_name, quoted_echo_app_name },
.{ echo_app_name, null },
.{ echo_app_name, "" },
.{ null, quoted_echo_app_name },
}) |case| {
constapp_name, constcmd_line=case;
constechoed_cmd_line=tryspawnChildReadStdout(allocator, app_name, cmd_line);
deferallocator.free(echoed_cmd_line);
std.debug.print("{s}\n", .{echoed_cmd_line});
}
}
extern"shell32"fnCommandLineToArgvW(lpCmdLine: windows.LPCWSTR, pNumArgs: *c_int) ?[*]windows.LPCWSTR;
fnspawnChildReadStdout(allocator: std.mem.Allocator, app_name: ?[]constu8, cmd_line: ?[]constu8) ![]u8 {
constchild_stdout, constchild_proc=spawn: {
constapp_name_w=if (app_name!=null)
trystd.unicode.utf8ToUtf16LeWithNull(allocator, app_name.?)
elsenull;
deferif (app_name_w!=null) allocator.free(app_name_w.?);
constcmd_line_w=if (cmd_line!=null)
trystd.unicode.utf8ToUtf16LeWithNull(allocator, cmd_line.?)
elsenull;
deferif (cmd_line_w!=null) allocator.free(cmd_line_w.?);
varchild_stdout_rd: windows.HANDLE=undefined;
varchild_stdout_wr: windows.HANDLE=undefined;
trywindows.CreatePipe(&child_stdout_rd, &child_stdout_wr, &.{
.nLength=@sizeOf(windows.SECURITY_ATTRIBUTES),
.lpSecurityDescriptor=null,
.bInheritHandle=windows.TRUE,
});
errdeferwindows.CloseHandle(child_stdout_rd);
deferwindows.CloseHandle(child_stdout_wr);
trywindows.SetHandleInformation(child_stdout_rd, windows.HANDLE_FLAG_INHERIT, 0);
varstartup_info: windows.STARTUPINFOW= .{
.cb=@sizeOf(windows.STARTUPINFOW),
.lpReserved=null,
.lpDesktop=null,
.lpTitle=null,
.dwX=0,
.dwY=0,
.dwXSize=0,
.dwYSize=0,
.dwXCountChars=0,
.dwYCountChars=0,
.dwFillAttribute=0,
.dwFlags=windows.STARTF_USESTDHANDLES,
.wShowWindow=0,
.cbReserved2=0,
.lpReserved2=null,
.hStdInput=null,
.hStdOutput=child_stdout_wr,
.hStdError=null,
};
varproc_info: windows.PROCESS_INFORMATION=undefined;
trywindows.CreateProcessW(
app_name_worelsenull,
cmd_line_worelsenull,
null,
null,
windows.TRUE,
0,
null,
null,
&startup_info,
&proc_info,
);
windows.CloseHandle(proc_info.hThread);
break :spawn .{ @as(std.fs.File, .{ .handle=child_stdout_rd }), proc_info.hProcess };
};
deferwindows.CloseHandle(child_proc);
deferwindows.WaitForSingleObjectEx(child_proc, windows.INFINITE, false) catch {};
returntrychild_stdout.reader().readAllAlloc(allocator, 4096);
}

zig run ./test_cmd_line.zig -- ./echo_cmd_line.exe

"./echo_cmd_line.exe"
"./echo_cmd_line.exe"
"./echo_cmd_line.exe"
"./echo_cmd_line.exe"

(I think it's worth noting that the output still uses forward slashes, which suggests that Windows doesn't really mess with the provided paths.)

Edit: I forgot to add, the CreateProcessW signature is currently incorrect, it should start with CreateProcessW(lpApplicationName: ?LPCWSTR, lpCommandLine: ?LPWSTR, ... so in order to compile test_cmd_line.zig you need to change the definitions. I'll open another PR for this.