From 4c2b682d14ef01ca82e42324d692d0a34575a860 Mon Sep 17 00:00:00 2001 From: Harsha Vardhan Simhadri Date: Sat, 15 May 2021 01:13:46 -0700 Subject: [PATCH 1/4] added tsv to bin format convertor --- include/distance.h | 4 +- tests/utils/CMakeLists.txt | 7 +++ tests/utils/compute_groundtruth.cpp | 27 +++-------- tests/utils/tsv_to_bin.cpp | 72 +++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 24 deletions(-) create mode 100644 tests/utils/tsv_to_bin.cpp diff --git a/include/distance.h b/include/distance.h index a4f311e510..3d403d1e56 100644 --- a/include/distance.h +++ b/include/distance.h @@ -226,9 +226,7 @@ namespace diskann { } }; - // Gopal. Slow implementations of the distance functions to get diskann to - // work in v14 machines that do not have AVX2 support. Performance here is not - // a concern, so we are using the simplest possible implementation. + // Slow implementations of the distance functions for machines without AVX2 template class SlowDistanceL2Int : public Distance { virtual float compare(const T *a, const T *b, unsigned length) const { diff --git a/tests/utils/CMakeLists.txt b/tests/utils/CMakeLists.txt index a343e377bc..6f7cb32b8e 100644 --- a/tests/utils/CMakeLists.txt +++ b/tests/utils/CMakeLists.txt @@ -17,6 +17,13 @@ if(MSVC) target_link_libraries(ivecs_to_bin optimized ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}/diskann_dll.lib) endif() +add_executable(tsv_to_bin tsv_to_bin.cpp) +if(MSVC) + target_link_options(tsv_to_bin PRIVATE /MACHINE:x64) + target_link_libraries(tsv_to_bin debug ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG}/diskann_dll.lib) + target_link_libraries(tsv_to_bin optimized ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}/diskann_dll.lib) +endif() + add_executable(int8_to_float int8_to_float.cpp) if(MSVC) target_link_options(int8_to_float PRIVATE /MACHINE:x64) diff --git a/tests/utils/compute_groundtruth.cpp b/tests/utils/compute_groundtruth.cpp index ed6f08b0b5..8fef8c929c 100644 --- a/tests/utils/compute_groundtruth.cpp +++ b/tests/utils/compute_groundtruth.cpp @@ -116,9 +116,7 @@ void exact_knn(const size_t dim, const size_t k, const float *const queries) // queries in Col major { float *points_l2sq = new float[npoints]; - // std::cout<<"jere"<(nptsuint64_t * ndimsuint64_t, ALIGNMENT); #pragma omp parallel for schedule(dynamic, 32768) for (int64_t i = 0; i < (int64_t) nptsuint64_t; i++) { @@ -278,11 +266,7 @@ inline void save_groundtruth_as_one_file(const std::string filename, template int aux_main(int argv, char **argc) { - if (argv != 6) { - command_line_help(); - return -1; - } - + size_t npoints, nqueries, dim; std::string base_file(argc[2]); std::string query_file(argc[3]); @@ -331,10 +315,6 @@ int aux_main(int argv, char **argc) { } } - // save_bin(gt_file + std::string("_ids.bin"), closest_points, nqueries, - // k); - // save_bin(gt_file + std::string("_dist.bin"), dist_closest_points, - // nqueries, k); save_groundtruth_as_one_file(gt_file, closest_points, dist_closest_points, nqueries, k); diskann::aligned_free(query_data); @@ -344,6 +324,11 @@ int aux_main(int argv, char **argc) { } int main(int argc, char **argv) { + if (argc != 6) { + command_line_help(); + return -1; + } + if (std::string(argv[1]) == std::string("float")) aux_main(argc, argv); if (std::string(argv[1]) == std::string("int8")) diff --git a/tests/utils/tsv_to_bin.cpp b/tests/utils/tsv_to_bin.cpp new file mode 100644 index 0000000000..111a6bb55d --- /dev/null +++ b/tests/utils/tsv_to_bin.cpp @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#include +#include "utils.h" + +template +void block_convert(std::ifstream& reader, std::ofstream& writer, _u64 npts, + _u64 ndims) { + auto read_buf = new T[4 * npts * (ndims + 1)]; + + auto cursor = read_buf; + T val; + + for (_u64 i = 0; i < npts; i++) { + for (_u64 d = 0; d < ndims; ++d) { + reader >> val; + *cursor = val; + cursor++; + } + } + writer.write((char*) read_buf, npts * ndims * sizeof(T)); + delete[] read_buf; +} + +int main(int argc, char** argv) { + if (argc != 6) { + std::cout << argv[0] + << " input_filename.tsv output_filename.bin dim num_pts>" + << std::endl; + exit(-1); + } + + if (std::string(argv[1]) != std::string("float") && + std::string(argv[1]) != std::string("int8") && + std::string(argv[1]) != std::string("uint8")) { + std::cout << "Unsupported type. float, int8 and uint8 types are supported." + << std::endl; + } + + _u64 ndims = atoi(argv[4]); + _u64 npts = atoi(argv[5]); + + std::ifstream reader(argv[2], std::ios::binary | std::ios::ate); + // _u64 fsize = reader.tellg(); + reader.seekg(0, std::ios::beg); + reader.seekg(0, std::ios::beg); + + _u64 blk_size = 131072; + _u64 nblks = ROUND_UP(npts, blk_size) / blk_size; + std::cout << "# blks: " << nblks << std::endl; + std::ofstream writer(argv[3], std::ios::binary); + auto npts_s32 = (_u32) npts; + auto ndims_s32 = (_u32) ndims; + writer.write((char*) &npts_s32, sizeof(_u32)); + writer.write((char*) &ndims_s32, sizeof(_u32)); + + for (_u64 i = 0; i < nblks; i++) { + _u64 cblk_size = std::min(npts - i * blk_size, blk_size); + if (std::string(argv[1]) == std::string("float")) { + block_convert(reader, writer, cblk_size, ndims); + } else if (std::string(argv[1]) == std::string("int8")) { + block_convert(reader, writer, cblk_size, ndims); + } else if (std::string(argv[1]) == std::string("uint8")) { + block_convert(reader, writer, cblk_size, ndims); + } + std::cout << "Block #" << i << " written" << std::endl; + } + + reader.close(); + writer.close(); +} From 361bca8b95c476672d84acf6b2c17ddbb2a42c8f Mon Sep 17 00:00:00 2001 From: Harsha Vardhan Simhadri Date: Fri, 4 Jun 2021 18:06:59 -0700 Subject: [PATCH 2/4] added tsv to bin convertor --- tests/utils/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/utils/CMakeLists.txt b/tests/utils/CMakeLists.txt index 6f7cb32b8e..9888f9679e 100644 --- a/tests/utils/CMakeLists.txt +++ b/tests/utils/CMakeLists.txt @@ -24,6 +24,13 @@ if(MSVC) target_link_libraries(tsv_to_bin optimized ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}/diskann_dll.lib) endif() +add_executable(bin_to_tsv bin_to_tsv.cpp) +if(MSVC) + target_link_options(bin_to_tsv PRIVATE /MACHINE:x64) + target_link_libraries(bin_to_tsv debug ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG}/diskann_dll.lib) + target_link_libraries(bin_to_tsv optimized ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}/diskann_dll.lib) +endif() + add_executable(int8_to_float int8_to_float.cpp) if(MSVC) target_link_options(int8_to_float PRIVATE /MACHINE:x64) From 27fc87875dec7243f433881bf9104dac919a6ddb Mon Sep 17 00:00:00 2001 From: Harsha Vardhan Simhadri Date: Fri, 4 Jun 2021 18:16:51 -0700 Subject: [PATCH 3/4] added tsv to bin convertor --- tests/utils/bin_to_tsv.cpp | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/utils/bin_to_tsv.cpp diff --git a/tests/utils/bin_to_tsv.cpp b/tests/utils/bin_to_tsv.cpp new file mode 100644 index 0000000000..37874e2437 --- /dev/null +++ b/tests/utils/bin_to_tsv.cpp @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#include +#include "utils.h" + +template +void block_convert(std::ofstream& writer, std::ifstream& reader, T* read_buf, + _u64 npts, _u64 ndims) { + reader.read((char*) read_buf, npts * ndims * sizeof(float)); + + for (_u64 i = 0; i < npts; i++) { + for (_u64 d = 0; d < ndims; d++) { + writer << read_buf[d + i * ndims]; + if (d < ndims - 1) + writer << "\t"; + else + writer << "\n"; + } + } +} + +int main(int argc, char** argv) { + if (argc != 4) { + std::cout << argv[0] << " input_bin output_tsv" << std::endl; + exit(-1); + } + std::string type_string(argv[1]); + if ((type_string != std::string("float")) && + (type_string != std::string("int8")) && + (type_string != std::string("uin8"))) { + std::cerr << "Error: type not supported. Use float/int8/uint8" << std::endl; + } + + std::ifstream reader(argv[2], std::ios::binary); + _u32 npts_u32; + _u32 ndims_u32; + reader.read((char*) &npts_u32, sizeof(_s32)); + reader.read((char*) &ndims_u32, sizeof(_s32)); + size_t npts = npts_u32; + size_t ndims = ndims_u32; + std::cout << "Dataset: #pts = " << npts << ", # dims = " << ndims + << std::endl; + + _u64 blk_size = 131072; + _u64 nblks = ROUND_UP(npts, blk_size) / blk_size; + + std::ofstream writer(argv[3]); + char* read_buf = new char[blk_size * ndims * 4]; + for (_u64 i = 0; i < nblks; i++) { + _u64 cblk_size = std::min(npts - i * blk_size, blk_size); + if (type_string == std::string("float")) + block_convert(writer, reader, (float*)read_buf, cblk_size, ndims); + else if (type_string == std::string("int8")) + block_convert(writer, reader, (int8_t*) read_buf, cblk_size, ndims); + else if (type_string == std::string("uint8")) + block_convert(writer, reader, (uint8_t*) read_buf, cblk_size, + ndims); + std::cout << "Block #" << i << " written" << std::endl; + } + + delete[] read_buf; + + writer.close(); + reader.close(); +} From 42754302385cdac2bb8293c26538e37c9aa8317e Mon Sep 17 00:00:00 2001 From: Harsha Vardhan Simhadri Date: Tue, 8 Jun 2021 16:20:52 -0700 Subject: [PATCH 4/4] added tool to convert float binary to int8 binary --- tests/utils/CMakeLists.txt | 7 ++++ tests/utils/float_bin_to_int8.cpp | 63 +++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/utils/float_bin_to_int8.cpp diff --git a/tests/utils/CMakeLists.txt b/tests/utils/CMakeLists.txt index 9888f9679e..e69722dcfc 100644 --- a/tests/utils/CMakeLists.txt +++ b/tests/utils/CMakeLists.txt @@ -10,6 +10,13 @@ if(MSVC) target_link_libraries(fvecs_to_bin optimized ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}/diskann_dll.lib) endif() +add_executable(float_bin_to_int8 float_bin_to_int8.cpp) +if(MSVC) + target_link_options(float_bin_to_int8 PRIVATE /MACHINE:x64) + target_link_libraries(float_bin_to_int8 debug ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG}/diskann_dll.lib) + target_link_libraries(float_bin_to_int8 optimized ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}/diskann_dll.lib) +endif() + add_executable(ivecs_to_bin ivecs_to_bin.cpp) if(MSVC) target_link_options(ivecs_to_bin PRIVATE /MACHINE:x64) diff --git a/tests/utils/float_bin_to_int8.cpp b/tests/utils/float_bin_to_int8.cpp new file mode 100644 index 0000000000..4f422a2336 --- /dev/null +++ b/tests/utils/float_bin_to_int8.cpp @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#include +#include "utils.h" + + +void block_convert(std::ofstream& writer, int8_t* write_buf, + std::ifstream& reader, float* read_buf, _u64 npts, + _u64 ndims, float bias, float scale) { + reader.read((char*) read_buf, npts * ndims * sizeof(float)); + + for (_u64 i = 0; i < npts; i++) { + for (_u64 d = 0; d < ndims; d++) { + write_buf[d + i * ndims] = + (int8_t)((read_buf[d + i * ndims] - bias) * (256.0 / scale)); + } + } + writer.write((char*) write_buf, npts * ndims); +} + +int main(int argc, char** argv) { + if (argc != 5) { + std::cout << "Usage: " << argv[0] << " input_bin output_tsv bias scale" + << std::endl; + exit(-1); + } + + std::ifstream reader(argv[1], std::ios::binary); + _u32 npts_u32; + _u32 ndims_u32; + reader.read((char*) &npts_u32, sizeof(_s32)); + reader.read((char*) &ndims_u32, sizeof(_s32)); + size_t npts = npts_u32; + size_t ndims = ndims_u32; + std::cout << "Dataset: #pts = " << npts << ", # dims = " << ndims + << std::endl; + + _u64 blk_size = 131072; + _u64 nblks = ROUND_UP(npts, blk_size) / blk_size; + + std::ofstream writer(argv[2], std::ios::binary); + auto read_buf = new float[blk_size * ndims]; + auto write_buf = new int8_t[blk_size * ndims]; + float bias = atof(argv[3]); + float scale = atof(argv[4]); + + writer.write((char*) (&npts_u32), sizeof(_u32)); + writer.write((char*) (&ndims_u32), sizeof(_u32)); + + for (_u64 i = 0; i < nblks; i++) { + _u64 cblk_size = std::min(npts - i * blk_size, blk_size); + block_convert(writer, write_buf, reader, read_buf, cblk_size, ndims, bias, + scale); + std::cout << "Block #" << i << " written" << std::endl; + } + + delete[] read_buf; + delete[] write_buf; + + writer.close(); + reader.close(); +}