Uh oh!
There was an error while loading. Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork 5.6k
Set of offset-based APIs for thread-safe file IO#53669
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
d37d1ee539393db60e876c61d639ad252de1540dbad8a311302cd5641430dd6ebaa92afbbcd4d7f94f8533ecb8df1af8f6a46da919ef23eb485eb93c8818aadc13de06c58252fb24f5c8baf0bd50aba1f143d31c1d41d652d8ac90bbbd26746d9b6fa946c5d4f807ef638e67afe8fbd1381776b1febba18db3f7af33d4caf8c84232126f15e613e7273f0751940609c891c2b3e462d49253faa1fd867356241b08648ab9be075441b1d300698eaf2476a4326dcd9e122b6a5554a6bb625ec9e1f4d002b2d5c9229994b1129405abe752ff9de609c065ae1dfbd4e5b09abc4b14043a926fc7616793db0891014688fde8c3405327b105e65ebc87354cf80bbb707051faFile filter
Filter by extension
Conversations
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| using System; | ||
| internal static partial class Interop | ||
| { | ||
| internal static partial class Sys | ||
| { | ||
| internal unsafe struct IOVector | ||
| { | ||
| public byte* Base; | ||
| public UIntPtr Count; | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| using System.Runtime.InteropServices; | ||
| internal static partial class Interop | ||
| { | ||
| internal static partial class Sys | ||
| { | ||
| [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PRead", SetLastError = true)] | ||
| internal static extern unsafe int PRead(SafeHandle fd, byte* buffer, int bufferSize, long fileOffset); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| using System.Runtime.InteropServices; | ||
| internal static partial class Interop | ||
| { | ||
| internal static partial class Sys | ||
| { | ||
| [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PReadV", SetLastError = true)] | ||
| internal static extern unsafe long PReadV(SafeHandle fd, IOVector* vectors, int vectorCount, long fileOffset); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| using System.Runtime.InteropServices; | ||
| internal static partial class Interop | ||
| { | ||
| internal static partial class Sys | ||
| { | ||
| [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PWrite", SetLastError = true)] | ||
| internal static extern unsafe int PWrite(SafeHandle fd, byte* buffer, int bufferSize, long fileOffset); | ||
adamsitnik marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| using System.Runtime.InteropServices; | ||
| internal static partial class Interop | ||
| { | ||
| internal static partial class Sys | ||
| { | ||
| [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PWriteV", SetLastError = true)] | ||
| internal static extern unsafe long PWriteV(SafeHandle fd, IOVector* vectors, int vectorCount, long fileOffset); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| using System; | ||
| using System.Runtime.InteropServices; | ||
| using System.Threading; | ||
| internal static partial class Interop | ||
| { | ||
| internal static partial class Kernel32 | ||
| { | ||
| [DllImport(Libraries.Kernel32, SetLastError = true)] | ||
| internal static extern unsafe int ReadFileScatter( | ||
| SafeHandle hFile, | ||
| long* aSegmentArray, | ||
| int nNumberOfBytesToRead, | ||
| IntPtr lpReserved, | ||
| NativeOverlapped* lpOverlapped); | ||
| [DllImport(Libraries.Kernel32, SetLastError = true)] | ||
| internal static extern unsafe int WriteFileGather( | ||
| SafeHandle hFile, | ||
| long* aSegmentArray, | ||
| int nNumberOfBytesToWrite, | ||
| IntPtr lpReserved, | ||
| NativeOverlapped* lpOverlapped); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -10,10 +10,6 @@ internal static partial class Interop | ||
| { | ||
| internal static partial class NtDll | ||
| { | ||
| internal const uint NT_ERROR_STATUS_DISK_FULL = 0xC000007F; | ||
| internal const uint NT_ERROR_STATUS_FILE_TOO_LARGE = 0xC0000904; | ||
| internal const uint NT_STATUS_INVALID_PARAMETER = 0xC000000D; | ||
| // https://msdn.microsoft.com/en-us/library/bb432380.aspx | ||
| // https://msdn.microsoft.com/en-us/library/windows/hardware/ff566424.aspx | ||
| [DllImport(Libraries.NtDll, CharSet = CharSet.Unicode, ExactSpelling = true)] | ||
| @@ -76,7 +72,7 @@ internal static unsafe (uint status, IntPtr handle) CreateFile( | ||
| } | ||
| } | ||
| internal static unsafe (uint status, IntPtr handle) CreateFile(ReadOnlySpan<char> path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) | ||
| internal static unsafe (uint status, IntPtr handle) NtCreateFile(ReadOnlySpan<char> path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) | ||
| { | ||
| // For mitigating local elevation of privilege attack through named pipes | ||
| // make sure we always call NtCreateFile with SECURITY_ANONYMOUS so that the | ||
| @@ -120,7 +116,7 @@ private static CreateDisposition GetCreateDisposition(FileMode mode) | ||
| private static DesiredAccess GetDesiredAccess(FileAccess access, FileMode fileMode, FileOptions options) | ||
| { | ||
| DesiredAccess result = 0; | ||
| DesiredAccess result = DesiredAccess.FILE_READ_ATTRIBUTES | DesiredAccess.SYNCHRONIZE; // default values used by CreateFileW | ||
| if ((access & FileAccess.Read) != 0) | ||
| { | ||
| @@ -134,13 +130,9 @@ private static DesiredAccess GetDesiredAccess(FileAccess access, FileMode fileMo | ||
| { | ||
| result |= DesiredAccess.FILE_APPEND_DATA; | ||
| } | ||
| if ((options & FileOptions.Asynchronous) == 0) | ||
| { | ||
| result |= DesiredAccess.SYNCHRONIZE; // required by FILE_SYNCHRONOUS_IO_NONALERT | ||
| } | ||
| if ((options & FileOptions.DeleteOnClose) != 0 || fileMode == FileMode.Create) | ||
adamsitnik marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| if ((options & FileOptions.DeleteOnClose) != 0) | ||
| { | ||
| result |= DesiredAccess.DELETE; // required by FILE_DELETE_ON_CLOSE and FILE_SUPERSEDE (which deletes a file if it exists) | ||
| result |= DesiredAccess.DELETE; // required by FILE_DELETE_ON_CLOSE | ||
| } | ||
| return result; | ||
| @@ -190,7 +182,8 @@ private static CreateOptions GetCreateOptions(FileOptions options) | ||
| } | ||
| private static ObjectAttributes GetObjectAttributes(FileShare share) | ||
| => (share & FileShare.Inheritable) != 0 ? ObjectAttributes.OBJ_INHERIT : 0; | ||
| => ObjectAttributes.OBJ_CASE_INSENSITIVE | // default value used by CreateFileW | ||
| ((share & FileShare.Inheritable) != 0 ? ObjectAttributes.OBJ_INHERIT : 0); | ||
| /// <summary> | ||
| /// File creation disposition when calling directly to NT APIs. | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -22,6 +22,7 @@ | ||
| #include <sys/file.h> | ||
| #include <sys/ioctl.h> | ||
| #include <sys/socket.h> | ||
| #include <sys/uio.h> | ||
| #include <syslog.h> | ||
| #include <termios.h> | ||
| #include <unistd.h> | ||
| @@ -1454,3 +1455,107 @@ int32_t SystemNative_ReadProcessStatusInfo(pid_t pid, ProcessStatus* processStat | ||
| return -1; | ||
| #endif // __sun | ||
| } | ||
| int32_t SystemNative_PRead(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) | ||
| { | ||
| assert(buffer != NULL); | ||
| assert(bufferSize >= 0); | ||
| ssize_t count; | ||
| while ((count = pread(ToFileDescriptor(fd), buffer, (uint32_t)bufferSize, (off_t)fileOffset)) < 0 && errno == EINTR); | ||
| assert(count >= -1 && count <= bufferSize); | ||
| return (int32_t)count; | ||
| } | ||
| int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) | ||
| { | ||
| assert(buffer != NULL); | ||
| assert(bufferSize >= 0); | ||
| ssize_t count; | ||
| while ((count = pwrite(ToFileDescriptor(fd), buffer, (uint32_t)bufferSize, (off_t)fileOffset)) < 0 && errno == EINTR); | ||
| assert(count >= -1 && count <= bufferSize); | ||
| return (int32_t)count; | ||
| } | ||
| int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) | ||
| { | ||
| assert(vectors != NULL); | ||
| assert(vectorCount >= 0); | ||
| int64_t count = 0; | ||
| int fileDescriptor = ToFileDescriptor(fd); | ||
| #if HAVE_PREADV && !defined(TARGET_WASM) // preadv is buggy on WASM | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is there an issue to go along with that bug? MemberAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @stephentoub I've not reported it yet. @vargaz when I was testing The most suprising thing is that it happens in the native layer (source code) @vargaz where should I report the bug? Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's open an issue in runtime for now and include the link here. | ||
| while ((count = preadv(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); | ||
| #else | ||
| int64_t current; | ||
| for (int i = 0; i < vectorCount; i++) | ||
| { | ||
| IOVector vector = vectors[i]; | ||
| while ((current = pread(fileDescriptor, vector.Base, vector.Count, (off_t)(fileOffset + count))) < 0 && errno == EINTR); | ||
| if (current < 0) | ||
| { | ||
| // if previous calls were succesfull, we return what we got so far | ||
| // otherwise, we return the error code | ||
| return count > 0 ? count : current; | ||
| } | ||
| count += current; | ||
| // Incomplete pread operation may happen for two reasons: | ||
| // a) We have reached EOF. | ||
| // b) The operation was interrupted by a signal handler. | ||
| // To mimic preadv, we stop on the first incomplete operation. | ||
| if (current != (int64_t)vector.Count) | ||
| { | ||
| return count; | ||
| } | ||
| } | ||
| #endif | ||
| assert(count >= -1); | ||
| return count; | ||
| } | ||
| int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) | ||
| { | ||
| assert(vectors != NULL); | ||
| assert(vectorCount >= 0); | ||
| int64_t count = 0; | ||
| int fileDescriptor = ToFileDescriptor(fd); | ||
| #if HAVE_PWRITEV && !defined(TARGET_WASM) // pwritev is buggy on WASM | ||
| while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); | ||
| #else | ||
| int64_t current; | ||
| for (int i = 0; i < vectorCount; i++) | ||
| { | ||
| IOVector vector = vectors[i]; | ||
| while ((current = pwrite(fileDescriptor, vector.Base, vector.Count, (off_t)(fileOffset + count))) < 0 && errno == EINTR); | ||
| if (current < 0) | ||
| { | ||
| // if previous calls were succesfull, we return what we got so far | ||
| // otherwise, we return the error code | ||
| return count > 0 ? count : current; | ||
| } | ||
| count += current; | ||
| // Incomplete pwrite operation may happen for few reasons: | ||
| // a) There was not enough space available or the file is too large for given file system. | ||
| // b) The operation was interrupted by a signal handler. | ||
| // To mimic pwritev, we stop on the first incomplete operation. | ||
| if (current != (int64_t)vector.Count) | ||
| { | ||
| return count; | ||
| } | ||
| } | ||
| #endif | ||
| assert(count >= -1); | ||
| return count; | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -41,6 +41,14 @@ typedef struct | ||||||||||||||
| // add more fields when needed. | ||||||||||||||
| } ProcessStatus; | ||||||||||||||
| // NOTE: the layout of this type is intended to exactly match the layout of a `struct iovec`. There are | ||||||||||||||
| // assertions in pal_networking.c that validate this. | ||||||||||||||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we move them to here? runtime/src/libraries/Native/Unix/System.Native/pal_networking.c Lines 163 to 168 in c87354c
| ||||||||||||||
| typedef struct | ||||||||||||||
| { | ||||||||||||||
| uint8_t* Base; | ||||||||||||||
| uintptr_t Count; | ||||||||||||||
| } IOVector; | ||||||||||||||
| /* Provide consistent access to nanosecond fields, if they exist. */ | ||||||||||||||
| /* Seconds are always available through st_atime, st_mtime, st_ctime. */ | ||||||||||||||
| @@ -730,3 +738,31 @@ PALEXPORT int32_t SystemNative_LChflagsCanSetHiddenFlag(void); | ||||||||||||||
| * Returns 1 if the process status was read; otherwise, 0. | ||||||||||||||
| */ | ||||||||||||||
| PALEXPORT int32_t SystemNative_ReadProcessStatusInfo(pid_t pid, ProcessStatus* processStatus); | ||||||||||||||
| /** | ||||||||||||||
| * Reads the number of bytes specified into the provided buffer from the specified, opened file descriptor at specified offset. | ||||||||||||||
| * | ||||||||||||||
| * Returns the number of bytes read on success; otherwise, -1 is returned an errno is set. | ||||||||||||||
| */ | ||||||||||||||
| PALEXPORT int32_t SystemNative_PRead(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); | ||||||||||||||
| /** | ||||||||||||||
| * Writes the number of bytes specified in the buffer into the specified, opened file descriptor at specified offset. | ||||||||||||||
| * | ||||||||||||||
| * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. | ||||||||||||||
| */ | ||||||||||||||
| PALEXPORT int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); | ||||||||||||||
| /** | ||||||||||||||
| * Reads the number of bytes specified into the provided buffers from the specified, opened file descriptor at specified offset. | ||||||||||||||
| * | ||||||||||||||
| * Returns the number of bytes read on success; otherwise, -1 is returned an errno is set. | ||||||||||||||
| */ | ||||||||||||||
| PALEXPORT int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset); | ||||||||||||||
| /** | ||||||||||||||
| * Writes the number of bytes specified in the buffers into the specified, opened file descriptor at specified offset. | ||||||||||||||
| * | ||||||||||||||
| * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. | ||||||||||||||
| */ | ||||||||||||||
| PALEXPORT int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset); | ||||||||||||||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: nuint