From 6ef7327177bd8f6090f632251d7fa90bf29bb4aa Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 5 Oct 2016 11:45:28 +0100 Subject: [PATCH 1/4] Conversion utf8 to utf16 and pretty-printing of Java strings Added two functions for utf8 to utf16 conversion function depending on whether we use little or big endian. Added a function utf16_little_endian_to_ascii to display nicely java strings as an ascii sequence. --- src/util/unicode.cpp | 40 ++++++++++++++++++++++++++++++++++++++++ src/util/unicode.h | 4 ++++ 2 files changed, 44 insertions(+) diff --git a/src/util/unicode.cpp b/src/util/unicode.cpp index 82acd36d4ee..a98a895e6b8 100644 --- a/src/util/unicode.cpp +++ b/src/util/unicode.cpp @@ -7,6 +7,8 @@ Author: Daniel Kroening, kroening@kroening.com \*******************************************************************/ #include +#include +#include #include "unicode.h" @@ -258,3 +260,41 @@ const char **narrow_argv(int argc, const wchar_t **argv_wide) return argv_narrow; } + +std::wstring utf8_to_utf16_big_endian(const std::string& in) +{ + std::wstring_convert > converter; + return converter.from_bytes(in); +} + +std::wstring utf8_to_utf16_little_endian(const std::string& in) +{ + const std::codecvt_mode mode=std::codecvt_mode::little_endian; + + // default largest value codecvt_utf8_utf16 reads without error is 0x10ffff + // see: http://en.cppreference.com/w/cpp/locale/codecvt_utf8_utf16 + const unsigned long maxcode=0x10ffff; + + typedef std::codecvt_utf8_utf16 codecvt_utf8_utf16t; + std::wstring_convert converter; + return converter.from_bytes(in); +} + +std::string utf16_little_endian_to_ascii(const std::wstring& in) +{ + std::string result; + std::locale loc; + for(const auto c : in) + { + if(c<=255 && isprint(c, loc)) + result+=(unsigned char)c; + else + { + result+="\\u"; + char hex[5]; + snprintf(hex, sizeof(hex), "%04x", (wchar_t)c); + result+=hex; + } + } + return result; +} diff --git a/src/util/unicode.h b/src/util/unicode.h index edad95039f0..1e5040344d0 100644 --- a/src/util/unicode.h +++ b/src/util/unicode.h @@ -22,6 +22,10 @@ std::wstring widen(const std::string &s); std::string utf32_to_utf8(const std::basic_string &s); std::string utf16_to_utf8(const std::basic_string &s); +std::wstring utf8_to_utf16_big_endian(const std::string&); +std::wstring utf8_to_utf16_little_endian(const std::string&); +std::string utf16_little_endian_to_ascii(const std::wstring& in); + const char **narrow_argv(int argc, const wchar_t **argv_wide); #endif // CPROVER_UTIL_UNICODE_H From 0541fcdc697c44085e37d743a5ca2f5b5291bf06 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Mon, 13 Feb 2017 13:03:11 +0000 Subject: [PATCH 2/4] Require libstdc++ from g++-5 to build with Clang --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6af73e31878..0e1e3abba67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ matrix: packages: - libwww-perl - clang-3.7 + - libstdc++-5-dev - libubsan0 before_install: - mkdir bin ; ln -s /usr/bin/clang-3.7 bin/gcc From e433cd45296e1f6da87dcd542c683eabf5f4cbea Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Mon, 13 Feb 2017 14:22:02 +0000 Subject: [PATCH 3/4] Rework utf16-to-ascii to avoid snprintf There isn't a standard snprintf in MS toolchains until VS 2015, so use C++ iostreams stuff instead. --- src/util/unicode.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/util/unicode.cpp b/src/util/unicode.cpp index a98a895e6b8..34727ab0816 100644 --- a/src/util/unicode.cpp +++ b/src/util/unicode.cpp @@ -9,6 +9,8 @@ Author: Daniel Kroening, kroening@kroening.com #include #include #include +#include +#include #include "unicode.h" @@ -282,19 +284,20 @@ std::wstring utf8_to_utf16_little_endian(const std::string& in) std::string utf16_little_endian_to_ascii(const std::wstring& in) { - std::string result; + std::ostringstream result; std::locale loc; for(const auto c : in) { if(c<=255 && isprint(c, loc)) - result+=(unsigned char)c; + result << (unsigned char)c; else { - result+="\\u"; - char hex[5]; - snprintf(hex, sizeof(hex), "%04x", (wchar_t)c); - result+=hex; + result << "\\u" + << std::hex + << std::setw(4) + << std::setfill('0') + << (unsigned int)c; } } - return result; + return result.str(); } From 927d54a914dd73e558544070ac40d3cfe7d67d67 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 22 Feb 2017 10:32:42 +0000 Subject: [PATCH 4/4] Style unicode.cpp/h No functional changes intended --- src/util/unicode.cpp | 37 +++++++++++++++++++++++++++++++++++++ src/util/unicode.h | 6 +++--- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/util/unicode.cpp b/src/util/unicode.cpp index 34727ab0816..1e280783aff 100644 --- a/src/util/unicode.cpp +++ b/src/util/unicode.cpp @@ -263,12 +263,36 @@ const char **narrow_argv(int argc, const wchar_t **argv_wide) return argv_narrow; } +/*******************************************************************\ + +Function: utf8_to_utf16_big_endian + + Inputs: String in UTF-8 format + + Outputs: String in UTF-16BE format + + Purpose: Note this requires g++-5 libstdc++ / libc++ / MSVC2010+ + +\*******************************************************************/ + std::wstring utf8_to_utf16_big_endian(const std::string& in) { std::wstring_convert > converter; return converter.from_bytes(in); } +/*******************************************************************\ + +Function: utf8_to_utf16_little_endian + + Inputs: String in UTF-8 format + + Outputs: String in UTF-16LE format + + Purpose: Note this requires g++-5 libstdc++ / libc++ / MSVC2010+ + +\*******************************************************************/ + std::wstring utf8_to_utf16_little_endian(const std::string& in) { const std::codecvt_mode mode=std::codecvt_mode::little_endian; @@ -282,6 +306,19 @@ std::wstring utf8_to_utf16_little_endian(const std::string& in) return converter.from_bytes(in); } +/*******************************************************************\ + +Function: utf16_little_endian_to_ascii + + Inputs: String in UTF-16LE format + + Outputs: String in US-ASCII format, with \uxxxx escapes for other + characters + + Purpose: + +\*******************************************************************/ + std::string utf16_little_endian_to_ascii(const std::wstring& in) { std::ostringstream result; diff --git a/src/util/unicode.h b/src/util/unicode.h index 1e5040344d0..c4bcab617d4 100644 --- a/src/util/unicode.h +++ b/src/util/unicode.h @@ -22,9 +22,9 @@ std::wstring widen(const std::string &s); std::string utf32_to_utf8(const std::basic_string &s); std::string utf16_to_utf8(const std::basic_string &s); -std::wstring utf8_to_utf16_big_endian(const std::string&); -std::wstring utf8_to_utf16_little_endian(const std::string&); -std::string utf16_little_endian_to_ascii(const std::wstring& in); +std::wstring utf8_to_utf16_big_endian(const std::string &); +std::wstring utf8_to_utf16_little_endian(const std::string &); +std::string utf16_little_endian_to_ascii(const std::wstring &in); const char **narrow_argv(int argc, const wchar_t **argv_wide);