ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all \u003cpre\u003e\u003ccode\u003e blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + '
Skip to content

ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length \u003e 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

ARROW-10287: [C++] Avoid std::random_device#8444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 3 commits into from
Closed

ARROW-10287: [C++] Avoid std::random_device #8444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff5fe05
ARROW-10287: [C++] Avoid std::random_device
pitrou Oct 12, 2020
2831af4
Fix conversion errors
pitrou Oct 12, 2020
0b80236
Fix name clashes under Valgrind
pitrou Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/arrow/io/slow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/io/util_internal.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand DownExpand Up@@ -64,8 +65,8 @@ void LatencyGenerator::Sleep() {
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency) {
auto seed = static_cast<int32_t>(std::random_device()());
return std::make_shared<LatencyGeneratorImpl>(average_latency, seed);
return std::make_shared<LatencyGeneratorImpl>(
average_latency, static_cast<int32_t>(::arrow::internal::GetRandomSeed()));
}

std::shared_ptr<LatencyGenerator> LatencyGenerator::Make(double average_latency,
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/testing/random.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ class Array;

namespace random {

using SeedType = std::random_device::result_type;
using SeedType = int32_t;
constexpr SeedType kSeedMax = std::numeric_limits<SeedType>::max();

class ARROW_TESTING_EXPORT RandomArrayGenerator {
Expand Down
40 changes: 32 additions & 8 deletions cpp/src/arrow/util/io_util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1467,14 +1467,8 @@ std::vector<NativePathString> GetPlatformTemporaryDirs() {

std::string MakeRandomName(int num_chars) {
static const std::string chars = "0123456789abcdefghijklmnopqrstuvwxyz";
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a PRNG instead.
static std::random_device::result_type seed = 42;
std::default_random_engine gen(seed++);
#else
std::random_device gen;
#endif
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(GetRandomSeed()));
std::uniform_int_distribution<int> dist(0, static_cast<int>(chars.length() - 1));

std::string s;
Expand All@@ -1484,6 +1478,7 @@ std::string MakeRandomName(int num_chars) {
}
return s;
}

} // namespace

Result<std::unique_ptr<TemporaryDir>> TemporaryDir::Make(const std::string& prefix) {
Expand DownExpand Up@@ -1593,5 +1588,34 @@ Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler)
return Status::OK();
}

namespace {

std::mt19937_64 GetSeedGenerator() {
// Initialize Mersenne Twister PRNG with a true random seed.
#ifdef ARROW_VALGRIND
// Valgrind can crash, hang or enter an infinite loop on std::random_device,
// use a crude initializer instead.
// Make sure to mix in process id to avoid clashes when parallel testing.
const uint8_t dummy = 0;
ARROW_UNUSED(dummy);
std::mt19937_64 seed_gen(reinterpret_cast<uintptr_t>(&dummy) ^
static_cast<uintptr_t>(getpid()));
#else
std::random_device true_random;
std::mt19937_64 seed_gen(static_cast<uint64_t>(true_random()) ^
(static_cast<uint64_t>(true_random()) << 32));
#endif
return seed_gen;
}

} // namespace

int64_t GetRandomSeed() {
// The process-global seed generator to aims to avoid calling std::random_device
// unless truly necessary (it can block on some systems, see ARROW-10287).
static auto seed_gen = GetSeedGenerator();
return static_cast<int64_t>(seed_gen());
}

} // namespace internal
} // namespace arrow
10 changes: 10 additions & 0 deletions cpp/src/arrow/util/io_util.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -347,11 +347,21 @@ class ARROW_EXPORT SignalHandler {
/// \brief Return the current handler for the given signal number.
ARROW_EXPORT
Result<SignalHandler> GetSignalHandler(int signum);

/// \brief Set a new handler for the given signal number.
///
/// The old signal handler is returned.
ARROW_EXPORT
Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);

/// \brief Get an unpredictable random seed
///
/// This function may be slightly costly, so should only be used to initialize
/// a PRNG, not to generate a large amount of random numbers.
/// It is better to use this function rather than std::random_device, unless
/// absolutely necessary (e.g. to generate a cryptographic secret).
ARROW_EXPORT
int64_t GetRandomSeed();

} // namespace internal
} // namespace arrow
8 changes: 4 additions & 4 deletions cpp/src/arrow/util/rle_encoding_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include "arrow/type.h"
#include "arrow/util/bit_stream_utils.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/io_util.h"
#include "arrow/util/rle_encoding.h"

namespace arrow {
Expand DownExpand Up@@ -418,14 +419,13 @@ TEST(BitRle, Random) {
std::vector<int> values(ngroups + max_group_size);

// prng setup
std::random_device rd;
const auto seed = ::arrow::internal::GetRandomSeed();
std::default_random_engine gen(
static_cast<std::default_random_engine::result_type>(seed));
std::uniform_int_distribution<int> dist(1, 20);

for (int iter = 0; iter < niters; ++iter) {
// generate a seed with device entropy
uint32_t seed = rd();
std::mt19937 gen(seed);

bool parity = 0;
values.resize(0);

Expand Down
4 changes: 2 additions & 2 deletions cpp/src/gandiva/random_generator_holder.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
#include <random>

#include "arrow/status.h"
#include "arrow/util/io_util.h"

#include "gandiva/function_holder.h"
#include "gandiva/node.h"
Expand All@@ -46,8 +47,7 @@ class GANDIVA_EXPORT RandomGeneratorHolder : public FunctionHolder {
}

RandomGeneratorHolder() : distribution_(0, 1) {
std::random_device rd;
generator_.seed(rd());
generator_.seed(::arrow::internal::GetRandomSeed());
}

std::mt19937_64 generator_;
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/gandiva/tests/generate_data.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,7 @@
#include <string>

#include "arrow/util/decimal.h"
#include "arrow/util/io_util.h"

#pragma once

Expand All@@ -35,18 +36,13 @@ class DataGenerator {

class Random {
public:
explicit Random(uint32_t seed = 100) : seed_(seed) {}
Random() : gen_(::arrow::internal::GetRandomSeed()) {}
explicit Random(uint64_t seed) : gen_(seed) {}

// This is 3 times faster than random_device
#ifndef _MSC_VER
int32_t next() { return rand_r(&seed_); }
#else
int32_t next() { return random_dev_(); }
#endif
int32_t next() { return gen_(); }

private:
uint32_t seed_;
std::random_device random_dev_;
std::default_random_engine gen_;
};

class Int32DataGenerator : public DataGenerator<int32_t> {
Expand Down