Scneario Test : Build Google Test and Taef Test based on preprocessor definition

cmake/winml_unittests.cmake

@@ -60,6 +60,7 @@ add_dependencies(winml_test_common
   winml_api
   winml_dll
 )
+target_compile_definitions(winml_test_common PRIVATE BUILD_GOOGLE_TEST)
 set_winml_target_properties(winml_test_common)
 
 file(GLOB winml_test_api_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/api/*.cpp")
@@ -82,6 +83,7 @@ add_winml_test(
   LIBS winml_test_common ${winml_test_scenario_libs}
 )
 target_precompiled_header(winml_test_scenario testPch.h)
+target_compile_definitions(winml_test_scenario PRIVATE BUILD_GOOGLE_TEST)
 set_target_properties(winml_test_scenario PROPERTIES LINK_FLAGS
   "/DELAYLOAD:d2d1.dll /DELAYLOAD:d3d11.dll /DELAYLOAD:dxgi.dll"
 )

winml/test/common/SqueezeNetValidator.cpp

@@ -2,12 +2,11 @@
 #include "protobufHelpers.h"
 #include "fileHelpers.h"
 #include "core/common/common.h"
-#include <gtest/gtest.h>
 #include <winrt/Windows.Media.h>
 #include <winrt/Windows.Graphics.Imaging.h>
 #include <winrt/Windows.Storage.h>
 #include <winrt/Windows.Storage.Streams.h>
-
+#include <iostream>
 // using namespace winrt::Windows::Foundation;
 using namespace winrt::Windows::AI::MachineLearning;
 using namespace winrt::Windows::Foundation::Collections;
@@ -35,12 +34,12 @@ static void BindImage(
 
     if (bindAsInspectable)
     {
-        EXPECT_NO_THROW(binding.Bind(name, frame));
+        WINML_EXPECT_NO_THROW(binding.Bind(name, frame));
     }
     else
     {
         auto imagetensor = ImageFeatureValue::CreateFromVideoFrame(frame);
-        EXPECT_NO_THROW(binding.Bind(name, imagetensor));
+        WINML_EXPECT_NO_THROW(binding.Bind(name, imagetensor));
     }
 }
 
@@ -50,15 +49,15 @@ static void BindTensor(
     ITensor inputTensor,
     bool bindAsInspectable = false)
 {
-    EXPECT_TRUE(inputTensor != nullptr);
+    WINML_EXPECT_TRUE(inputTensor != nullptr);
 
     if (bindAsInspectable)
     {
-        EXPECT_NO_THROW(binding.Bind(name, inputTensor.as<TensorFloat>().GetAsVectorView()));
+        WINML_EXPECT_NO_THROW(binding.Bind(name, inputTensor.as<TensorFloat>().GetAsVectorView()));
     }
     else
     {
-        EXPECT_NO_THROW(binding.Bind(name, inputTensor));
+        WINML_EXPECT_NO_THROW(binding.Bind(name, inputTensor));
     }
 }
 
@@ -75,11 +74,11 @@ ITensor BindOutput(
     {
     case OutputBindingStrategy::Bound:
         outputTensor = T::Create(shape);
-        EXPECT_NO_THROW(binding.Bind(name, outputTensor));
+        WINML_EXPECT_NO_THROW(binding.Bind(name, outputTensor));
         break;
     case OutputBindingStrategy::Empty:
         outputTensor = T::Create();
-        EXPECT_NO_THROW(binding.Bind(name, outputTensor));
+        WINML_EXPECT_NO_THROW(binding.Bind(name, outputTensor));
         break;
     case OutputBindingStrategy::Unbound:
         __fallthrough;
@@ -104,7 +103,7 @@ ImageFeatureValue BindImageOutput(
         SoftwareBitmap bitmap(BitmapPixelFormat::Bgra8, 720, 720);
         VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(bitmap);
         outputTensor = ImageFeatureValue::CreateFromVideoFrame(frame);
-        EXPECT_NO_THROW(binding.Bind(name, outputTensor));
+        WINML_EXPECT_NO_THROW(binding.Bind(name, outputTensor));
         break;
     }
     case OutputBindingStrategy::Unbound:
@@ -136,26 +135,26 @@ void ModelValidator::FnsCandy16(
 
     // WinML model creation
     LearningModel model = nullptr;
-    EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath));
+    WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath));
 
     LearningModelSession modelSession = nullptr;
-    EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind)));
+    WINML_EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind)));
 
     LearningModelBinding modelBinding(modelSession);
     auto fullImagePath = modulePath + inputDataImageFileName;
     BindImage(modelBinding, inputBindingName, fullImagePath.c_str(), bindInputsAsIInspectable);
 
     // create the tensor for the actual output
     auto output = model.OutputFeatures().First().Current();
-    EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor);
+    WINML_EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor);
 
     auto shape = winrt::single_threaded_vector(std::vector<int64_t> {1, 1});
     auto outputTensor = BindImageOutput(outputBindingStrategy, modelBinding, outputDataBindingName);
 
     // Evaluate the model
     std::cout << "Calling EvaluateSync on instance" << instance << "\n";
     LearningModelEvaluationResult result = nullptr;
-    EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {}));
+    WINML_EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {}));
 
     // Get results
     if (outputBindingStrategy == OutputBindingStrategy::Unbound)
@@ -167,7 +166,7 @@ void ModelValidator::FnsCandy16(
     }
     else
     {
-        EXPECT_EQ(result.Outputs().Lookup(outputDataBindingName), outputTensor);
+        WINML_EXPECT_EQUAL(result.Outputs().Lookup(outputDataBindingName), outputTensor);
 
         auto softwareBitmap = outputTensor.VideoFrame().SoftwareBitmap();
 
@@ -203,10 +202,10 @@ void ModelValidator::SqueezeNet(
 
     // WinML model creation
     LearningModel model = nullptr;
-    EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath));
+    WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath));
 
     LearningModelSession modelSession = nullptr;
-    EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind)));
+    WINML_EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind)));
 
     LearningModelBinding modelBinding(modelSession);
 
@@ -224,19 +223,19 @@ void ModelValidator::SqueezeNet(
 
     // load up the expected output
     auto expectedResultsTensor = ProtobufHelpers::LoadTensorFromProtobufFile(outputFileName, false);
-    EXPECT_TRUE(expectedResultsTensor != nullptr);
+    WINML_EXPECT_TRUE(expectedResultsTensor != nullptr);
 
     // create the tensor for the actual output
     auto output = model.OutputFeatures().First().Current();
-    EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor);
+    WINML_EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor);
 
     auto outputTensor = BindOutput<TensorFloat>(
         outputBindingStrategy, modelBinding, outputDataBindingName, expectedResultsTensor.Shape());
 
     // Evaluate the model
     std::cout << "Calling EvaluateSync on instance" << instance << "\n";
     LearningModelEvaluationResult result = nullptr;
-    EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {}));
+    WINML_EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {}));
 
     // Get results
     if (outputBindingStrategy == OutputBindingStrategy::Unbound)
@@ -247,21 +246,22 @@ void ModelValidator::SqueezeNet(
     }
     else
     {
-        EXPECT_EQ(result.Outputs().Lookup(outputDataBindingName), outputTensor);
+        WINML_EXPECT_EQUAL(result.Outputs().Lookup(outputDataBindingName), outputTensor);
     }
 
     auto outDataExpected = expectedResultsTensor.as<TensorFloat>().GetAsVectorView();
     auto outDataActual = outputTensor.as<TensorFloat>().GetAsVectorView();
 
-    EXPECT_TRUE(outDataActual.Size() == outDataExpected.Size());
+    WINML_EXPECT_TRUE(outDataActual.Size() == outDataExpected.Size());
     for (uint32_t i = 0; i < outDataActual.Size(); i++)
     {
         float delta = std::abs(outDataActual.GetAt(i) - outDataExpected.GetAt(i));
         if (delta > dataTolerance)
         {
-            ADD_FAILURE() << "EXPECTED: " << outDataExpected.GetAt(i) << " , ACTUAL: " << outDataActual.GetAt(i)
+          std::stringstream ss;
+          ss << "EXPECTED: " << outDataExpected.GetAt(i) << " , ACTUAL: " << outDataActual.GetAt(i)
                 << "instance " << instance << ", element " << i;
-
+          WINML_LOG_ERROR(ss.str().c_str());
         }
     }
 }

winml/test/common/SqueezeNetValidator.h

@@ -6,7 +6,7 @@
 
 #pragma once
 
-#include <std.h>
+#include "std.h"
 
 enum OutputBindingStrategy { Bound, Unbound, Empty };
 

winml/test/common/googleTestMacros.h

@@ -0,0 +1,63 @@
+#include <gtest/gtest.h>
+#include "runtimeParameters.h"
+
+#define TEST_GROUP_BEGIN(group_name)
+#define TEST_GROUP_END()
+
+#define WINML_TEST(group_name, test_name) \
+  TEST_F(group_name, test_name) {         \
+    getapi().test_name();                 \
+  }
+
+#define WINML_TEST_CLASS_BEGIN_NO_SETUP(test_class_name) \
+  class test_class_name : public ::testing::Test {       \
+  };
+
+#define WINML_TEST_CLASS_BEGIN_WITH_SETUP(test_class_name, setup_method) \
+  class test_class_name : public ::testing::Test {                       \
+   protected:                                                            \
+    void SetUp() override {                                              \
+      getapi().setup_method();                                           \
+    }                                                                    \
+  };
+
+#define WINML_TEST_CLASS_END()
+
+// For old versions of gtest without GTEST_SKIP, stream the message and return success instead
+#ifndef GTEST_SKIP
+#define GTEST_SKIP_(message) \
+  return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess)
+#define GTEST_SKIP GTEST_SKIP_("")
+#endif
+
+#define WINML_SKIP_TEST(message) \
+  GTEST_SKIP() << message;
+
+#define WINML_EXPECT_NO_THROW(statement) EXPECT_NO_THROW(statement)
+#define WINML_EXPECT_TRUE(statement) EXPECT_TRUE(statement)
+#define WINML_EXPECT_EQUAL(val1, val2) EXPECT_EQ(val1, val2)
+#define WINML_EXPECT_NOT_EQUAL(val1, val2) EXPECT_NE(val1, val2)
+
+#define WINML_LOG_ERROR(message) \
+  ADD_FAILURE() << message
+
+#define WINML_EXPECT_HRESULT_SUCCEEDED(hresult_expression) EXPECT_HRESULT_SUCCEEDED(hresult_expression)
+#define WINML_EXPECT_HRESULT_FAILED(hresult_expression) EXPECT_HRESULT_FAILED(hresult_expression)
+#define WINML_EXPECT_THROW_SPECIFIC(statement, exception, condition) EXPECT_THROW_SPECIFIC(statement, exception, condition)
+
+#ifndef USE_DML
+#define GPUTEST \
+  WINML_SKIP_TEST("GPU tests disabled because this is a WinML only build (no DML)")
+#else
+#define GPUTEST                                                                         \
+  if (auto noGpuTests = RuntimeParameters::Parameters.find("noGPUtests");               \
+      noGpuTests != RuntimeParameters::Parameters.end() && noGpuTests->second != "0") { \
+    WINML_SKIP_TEST("GPU tests disabled");                                              \
+  }
+#endif
+
+#define SKIP_EDGECORE                                                                   \
+  if (auto isEdgeCore = RuntimeParameters::Parameters.find("EdgeCore");                 \
+      isEdgeCore != RuntimeParameters::Parameters.end() && isEdgeCore->second != "0") { \
+    WINML_SKIP_TEST("Test can't be run in EdgeCore");                                   \
+  }

winml/test/common/protobufHelpers.cpp

@@ -1,18 +1,18 @@
-#define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS
+#ifndef _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS
+#define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS
+#endif
 
 // LotusRT
 #include "core/framework/allocatormgr.h"
 #include "core/common/logging/logging.h"
 #include "core/common/logging/sinks/clog_sink.h"
-
 #include "protobufHelpers.h"
 
 #pragma warning(push)
 #pragma warning(disable : 4100)
 #include "onnx/onnx-ml.pb.h"
 #pragma warning(pop)
 
-#include <gtest/gtest.h>
 #include <fstream>
 
 #include "winrt/Windows.Storage.Streams.h"
@@ -66,33 +66,35 @@ bool LoadTensorFromPb(onnx::TensorProto& tensor, std::wstring filePath) {
 
 template <typename DataType>
 std::vector<DataType> GetTypeSpecificDataFromTensorProto(
-  onnx::TensorProto /*tensorProto*/){
+    onnx::TensorProto /*tensorProto*/) {
   static_assert(false, "UNDEFINED! TensorProto methods aren't templated, so add a new template specialization.");
 }
 template <>
 std::vector<float> GetTypeSpecificDataFromTensorProto(
-  onnx::TensorProto tensorProto){
+    onnx::TensorProto tensorProto) {
   return std::vector<float>(std::begin(tensorProto.float_data()), std::end(tensorProto.float_data()));
 }
 template <>
 std::vector<int32_t> GetTypeSpecificDataFromTensorProto(
-  onnx::TensorProto tensorProto){
+    onnx::TensorProto tensorProto) {
   return std::vector<int32_t>(std::begin(tensorProto.int32_data()), std::end(tensorProto.int32_data()));
 }
 template <>
 std::vector<int64_t> GetTypeSpecificDataFromTensorProto(
-  onnx::TensorProto tensorProto){
+    onnx::TensorProto tensorProto) {
   return std::vector<int64_t>(std::begin(tensorProto.int64_data()), std::end(tensorProto.int64_data()));
 }
 
 template <typename DataType>
 std::vector<DataType> GetTensorDataFromTensorProto(
-	onnx::TensorProto tensorProto,
-	uint64_t elementCount) {
+    onnx::TensorProto tensorProto,
+    uint64_t elementCount) {
   if (tensorProto.has_raw_data()) {
     std::vector<DataType> tensorData;
     auto& values = tensorProto.raw_data();
-    EXPECT_EQ(elementCount, values.size() / sizeof(DataType)) << L"TensorProto elementcount should match raw data buffer size in elements.";
+    if (elementCount != values.size() / sizeof(DataType)) {
+        WINML_LOG_ERROR("TensorProto element count should match raw data buffer size in elements.");
+    }
 
     tensorData = std::vector<DataType>(elementCount);
     memcpy(tensorData.data(), values.data(), values.size());
@@ -105,7 +107,7 @@ std::vector<DataType> GetTensorDataFromTensorProto(
 static std::vector<winrt::hstring> GetTensorStringDataFromTensorProto(
     onnx::TensorProto tensorProto,
     uint64_t elementCount) {
-  EXPECT_EQ(tensorProto.string_data_size(), elementCount);
+  WINML_EXPECT_EQUAL(tensorProto.string_data_size(), elementCount);
   auto& values = tensorProto.string_data();
   auto returnVector = std::vector<winrt::hstring>(elementCount);
   std::transform(std::begin(values), std::end(values), std::begin(returnVector),
@@ -131,15 +133,15 @@ ITensor ProtobufHelpers::LoadTensorFromProtobufFile(
     }
     switch (tensorProto.data_type()) {
       case (onnx::TensorProto::DataType::TensorProto_DataType_FLOAT):
-       return TensorFloat::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto<float>(tensorProto, elementCount));
+        return TensorFloat::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto<float>(tensorProto, elementCount));
       case (onnx::TensorProto::DataType::TensorProto_DataType_INT32):
-       return TensorInt32Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto<int32_t>(tensorProto, elementCount));
+        return TensorInt32Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto<int32_t>(tensorProto, elementCount));
       case (onnx::TensorProto::DataType::TensorProto_DataType_INT64):
-       return TensorInt64Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto<int64_t>(tensorProto, elementCount));
+        return TensorInt64Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto<int64_t>(tensorProto, elementCount));
       case (onnx::TensorProto::DataType::TensorProto_DataType_STRING):
         return TensorString::CreateFromIterable(tensorShape, GetTensorStringDataFromTensorProto(tensorProto, elementCount));
       default:
-        ADD_FAILURE() << L"Tensor type for creating tensor from protobuf file not supported.";
+        WINML_LOG_ERROR("Tensor type for creating tensor from protobuf file not supported.");
         break;
     }
   }
@@ -152,7 +154,7 @@ TensorFloat16Bit ProtobufHelpers::LoadTensorFloat16FromProtobufFile(
   onnx::TensorProto tensorProto;
   if (LoadTensorFromPb(tensorProto, filePath)) {
     if (tensorProto.has_data_type()) {
-      EXPECT_EQ(onnx::TensorProto::DataType::TensorProto_DataType_FLOAT16, tensorProto.data_type());
+      WINML_EXPECT_EQUAL(onnx::TensorProto::DataType::TensorProto_DataType_FLOAT16, tensorProto.data_type());
     } else {
       std::cerr << "Loading unknown TensorProto datatype as TensorFloat16Bit.\n";
     }
@@ -166,7 +168,10 @@ TensorFloat16Bit ProtobufHelpers::LoadTensorFloat16FromProtobufFile(
     uint32_t sizeInBytes;
     spTensorValueNative->GetBuffer(reinterpret_cast<BYTE**>(&data), &sizeInBytes);
 
-    EXPECT_TRUE(tensorProto.has_raw_data()) << L"Float16 tensor proto buffers are expected to contain raw data.";
+    if (!tensorProto.has_raw_data())
+    {
+      WINML_LOG_ERROR("Float16 tensor proto buffers are expected to contain raw data.");
+    }
 
     auto& raw_data = tensorProto.raw_data();
     auto buff = raw_data.c_str();

winml/test/common/runtimeParameters.h

@@ -6,4 +6,4 @@ namespace RuntimeParameters
 {
     // Runtime parameters passed through CLI arguments
     extern std::unordered_map<std::string, std::string> Parameters;
-}
+}