Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
*.tar.gz
1 change: 1 addition & 0 deletions clickhouse/.gitignore
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
clickhouse*
41 changes: 41 additions & 0 deletions clickhouse/README.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# clickhouse

Patches and notes for building ClickHouse on illumos.

## Building

```bash
$ ./build.sh
```

## Patches

Most patches have been upstreamed into their various homes. The remaining are
mostly related to some C++ standard library wierdness. The files in
`patches/direct` are applied to the source (after cloning submodules), and
those in `patches/cmake` are applied after running `cmake` as they apply to
some of the generated build files.

## Upstreaming

In general, ClickHouse was very responsive to PRs, so additional work to
upstream things should be straightforward. The only bit to record is how to
handle updates to any of the repos cloned as submodules.

After things have been upstreamed, say to `contrib/project-a`, run

```bash
$ git submodule update --checkout --remote contrib/project-a
```

This records the latest commit of the remote submodule into the superproject.
Once all submodules have been updated like this, make a commit and put up
a PR against ClickHouse as usual.

## Errors

If you see errors complaining about a submodule not having a particular
commit or branch, you may need to specify the `branch` in the `.gitmodules`
file. It should be whatever the remote uses as the default branch. Git
defaults to using `master`, but many remotes don't use that name, hence
the errors.
162 changes: 162 additions & 0 deletions clickhouse/build.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
#!/bin/bash

set -o errexit
set -o pipefail

function info {
printf 'INFO: %s\n' "$*"
}

function header {
printf -- '\n'
printf -- '----------------------------------------------------------\n'
printf -- 'INFO: %s\n' "$*"
printf -- '----------------------------------------------------------\n'
printf -- '\n'
}

function fatal {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

ROOT=$(cd "$(dirname "$0")" && pwd)
ARTEFACT="$ROOT/clickhouse"
WORK="$ARTEFACT/build"
VER="21.7"
BRANCH="master"
REPO="https://github.com/oxidecomputer/clickhouse"

# Get platform specific options/tools/paths
if [ $# -eq 0 ]; then
PLATFORM="$OSTYPE"
else
PLATFORM="$1"
fi
case $PLATFORM in
linux*)
PLATFORM="linux"
BUILD_COMMAND="make"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="--strip-debug"
NPROC="$(nproc)"
;;
darwin*)
PLATFORM="macos"
BUILD_COMMAND="make"
CC=clang
CXX=clang
STRIP_ARGS="-S"
NPROC="$(sysctl -n hw.ncpu)"
;;
solaris*|illumos*)
PLATFORM="illumos"
BUILD_COMMAND="ninja"
CC=gcc-10
CXX=g++-10
STRIP_ARGS="-x"
NPROC="$(nproc)"
;;
*)
failed "Unsupported platform $PLATFORM"
exit 1
;;
esac
COMMON_PATCH_DIR="$ROOT/common/patches"
PATCH_DIR="$ROOT/$PLATFORM/patches"
FILES_DIR="$ROOT/$PLATFORM/files"
EXTRA_FILES=""
if [ -d "$FILES_DIR" ]; then
EXTRA_FILES="$(ls "$FILES_DIR")"
fi
header "Building clickhouse for $PLATFORM"

#
# Download ClickHouse sources
#
if [ -d "$ARTEFACT" ]; then
info "ClickHouse repo exists, resetting to HEAD"
cd "$ARTEFACT"
git fetch origin
git switch "$BRANCH"
git reset --hard "origin/$BRANCH"
git submodule update --checkout --recursive --force
else
info "Cloning ClickHouse sources"
git clone "$REPO"
cd "$ARTEFACT"
git switch "$BRANCH"
git submodule update --init --recursive
fi

# Apply common patches, independent of platform
header "Applying shared ClickHouse patches"
git apply --verbose $COMMON_PATCH_DIR/*

# Patches to the actual sources. Below we apply those to CMake-generated files.
if [ -d "$PATCH_DIR/direct" ]; then
header "Applying $PLATFORM-specific patches"
git apply --verbose $PATCH_DIR/direct/*
fi

header "Building ClickHouse"
mkdir -p "$WORK" && cd "$WORK"
FLAGS="-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -D__EXTENSIONS__ -m64 -I$ARTEFACT/contrib/hyperscan-cmake/x86_64/"
CC=$CC CXX=$CXX CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" \
cmake \
-DABSL_CXX_STANDARD="17" \
-DENABLE_LDAP=off \
-DUSE_INTERNAL_LDAP_LIBRARY=off \
-DENABLE_HDFS=off \
-DUSE_INTERNAL_HDFS3_LIBRARY=off \
-DENABLE_AMQPCPP=off \
-DENABLE_AVRO=off \
-DUSE_INTERNAL_AVRO_LIBRARY=off \
-DENABLE_CAPNP=off \
-DUSE_INTERNAL_CAPNP_LIBRARY=off \
-DENABLE_MSGPACK=off \
-DUSE_INTERNAL_MSGPACK_LIBRARY=off \
-DENABLE_MYSQL=off \
-DENABLE_S3=off \
-DUSE_INTERNAL_AWS_S3_LIBRARY=off \
-DENABLE_PARQUET=off \
-DUSE_INTERNAL_PARQUET_LIBRARY=off \
-DENABLE_ORC=off \
-DUSE_INTERNAL_ORC_LIBRARY=off \
-DUSE_SENTRY=off \
-DENABLE_CLICKHOUSE_ODBC_BRIDGE=off \
-DENABLE_CLICKHOUSE_BENCHMARK=off \
-DENABLE_TESTS=off \
"$ARTEFACT"

header "Patching CMake-generated files"
cd "$ARTEFACT"
if [ -d "$PATCH_DIR/cmake" ]; then
git apply --verbose $PATCH_DIR/cmake/*
fi
cd "$WORK"

# The build is massive. Try to parallelize until we error out, usually due to space constraints while
# linking. At that point, continue serially
$BUILD_COMMAND -j "$NPROC" || (header "Parallel build failed, continuing serially" && $BUILD_COMMAND -j 1)

# Strip the resulting binary. This part is crucial. ClickHouse's binary is 3+GiB unstripped.
strip $STRIP_ARGS "$WORK/programs/clickhouse"
CONFIG_FILE_DIR="$ARTEFACT/programs/server"
CONFIG_FILE_NAME="config.xml"
if [ -z "$EXTRA_FILES" ]; then
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME"
else
/usr/bin/tar cvfz \
$ROOT/clickhouse-v$VER.$PLATFORM.tar.gz \
-C "$WORK/programs" clickhouse \
-C "$CONFIG_FILE_DIR" "$CONFIG_FILE_NAME" \
-C "$FILES_DIR" "$EXTRA_FILES"
fi

header "Build output:"
find "$WORK" -type f -ls
64 changes: 64 additions & 0 deletions clickhouse/common/patches/config.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
diff --git a/programs/server/config.xml b/programs/server/config.xml
index df8a5266c3..e58318e970 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -22,8 +22,8 @@
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
-->
<level>trace</level>
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <log>/tmp/clickhouse/21.7/clickhouse-server.log</log>
+ <errorlog>/tmp/clickhouse/21.7/clickhouse-server.err.log</errorlog>
<!-- Rotation policy
See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
-->
@@ -333,10 +333,10 @@
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>

<!-- Path to data directory, with trailing slash. -->
- <path>/var/lib/clickhouse/</path>
+ <path>/tmp/clickhouse/21.7/</path>

<!-- Path to temporary data for processing hard queries. -->
- <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+ <tmp_path>/tmp/clickhouse/21.7/tmp/</tmp_path>

<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
@@ -350,7 +350,7 @@
<!-- <tmp_policy>tmp</tmp_policy> -->

<!-- Directory with user provided files that are accessible by 'file' table function. -->
- <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+ <user_files_path>/tmp/clickhouse/21.7/user_files/</user_files_path>

<!-- LDAP server definitions. -->
<ldap_servers>
@@ -425,7 +425,7 @@
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
- <path>/var/lib/clickhouse/access/</path>
+ <path>/tmp/clickhouse/21.7/access/</path>
</local_directory>

<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
@@ -927,7 +927,7 @@
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->


- <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+ <!-- <top_level_domains_path>/tmp/clickhouse/21.7/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>

@@ -1040,7 +1040,7 @@
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
- <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+ <format_schema_path>/tmp/clickhouse/21.7/format_schemas/</format_schema_path>

<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
13 changes: 13 additions & 0 deletions clickhouse/common/patches/embedded.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml
index a66f57d1eb..bfb374091c 100644
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@@ -10,7 +10,7 @@
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>

- <path>./</path>
+ <path>/tmp</path>

<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
35 changes: 35 additions & 0 deletions clickhouse/illumos/files/manifest.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='clickhouse'>

<service name='system/illumos/clickhouse' type='service' version='1'>
<create_default_instance enabled='false' />
<single_instance />

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/clickhouse/21.7/clickhouse server --config-file %{config/config_file} &amp;'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='config_file' type='astring' value='/opt/clickhouse/21.7/config.xml' />
</property_group>

<property_group name='startd' type='framework'>
<propval name='duration' type='astring' value='contract' />
</property_group>

<stability value='Unstable' />

<template>
<common_name>
<loctext xml:lang='C'>ClickHouse</loctext>
</common_name>
<description>
<loctext xml:lang='C'>Oxide metric database</loctext>
</description>
</template>
</service>

</service_bundle>
18 changes: 18 additions & 0 deletions clickhouse/illumos/patches/cmake/boringssl-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a6..7e68e7c 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -78,7 +78,11 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
- set(ARCH "x86")
+ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+ set(ARCH "x86_64")
+ else()
+ set(ARCH "x86")
+ endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
set(ARCH "x86")
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
Submodule contrib/cppkafka contains modified content
16 changes: 16 additions & 0 deletions clickhouse/illumos/patches/cmake/poco-cmake.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt
index 9bc06e5..dd6894f 100644
--- a/contrib/poco-cmake/Net/CMakeLists.txt
+++ b/contrib/poco-cmake/Net/CMakeLists.txt
@@ -127,6 +127,10 @@ if (USE_INTERNAL_POCO_LIBRARY)
)
target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include")
target_link_libraries (_poco_net PUBLIC Poco::Foundation)
+
+ if (OS_SUNOS)
+ target_link_libraries (_poco_net PUBLIC socket nsl)
+ endif()
else ()
add_library (Poco::Net UNKNOWN IMPORTED GLOBAL)

Submodule contrib/rocksdb contains modified content
10 changes: 10 additions & 0 deletions clickhouse/illumos/patches/direct/AbseilConfigureCopts.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
diff --git a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
index 77d4ace..30891a4 100644
--- a/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
+++ b/contrib/abseil-cpp/absl/copts/AbseilConfigureCopts.cmake
@@ -72,3 +72,5 @@ elseif(NOT "${CMAKE_CXX_STANDARD}")
else()
set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}")
endif()
+# Absl cannot work with C++20, which removed some typedefs for std::allocator
+set(ABSL_CXX_STANDARD 17)
13 changes: 13 additions & 0 deletions clickhouse/illumos/patches/direct/PerformanceAdaptors.patch
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h
index 5d4d013..716bb65 100644
--- a/src/Functions/PerformanceAdaptors.h
+++ b/src/Functions/PerformanceAdaptors.h
@@ -99,7 +99,7 @@ namespace detail
/// when there is no statistical significant difference between them.
double sigma() const
{
- return mean() / sqrt(adjustedCount());
+ return mean() / sqrt(static_cast<double>(adjustedCount()));
}

void run()
Loading