Microsoft CDM Release Version Update: 1.0.13

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.ALL.nuspec

@@ -5,7 +5,7 @@
   <metadata>
 
     <id>Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.All</id>
-    <version>1.0.12</version>
+    <version>1.0.13</version>
     <description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</description>
 
     <authors>Microsoft Corporation.</authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>netstandard2.0;net45;net462</TargetFrameworks>
-    <version>1.0.12</version>
+    <version>1.0.13</version>
     <Description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</Description>
 
     <Authors>Microsoft Corporation</Authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.nuspec

@@ -5,7 +5,7 @@
   <metadata>
 
     <id>Microsoft.CommonDataModel.ObjectModel.Adapter.Adls</id>
-    <version>1.0.12</version>
+    <version>1.0.13</version>
     <description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</description>
 
     <authors>Microsoft Corporation.</authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/AdlsModelJsonTestHelper.cs

@@ -0,0 +1,104 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using Microsoft.CommonDataModel.ObjectModel.Storage;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.IO;
+using System.Security.Cryptography.X509Certificates;
+
+namespace Microsoft.CommonDataModel.ObjectModel.Tests
+{
+    /// <summary>
+    /// Class for ADLS based model.json tests.
+    /// </summary>
+    public static class AdlsModelJsonTestHelper
+    {
+        public static readonly string actualFolderName = "Actual";
+        public static readonly string expectedFolderName = "Expected";
+        public static readonly string inputFolderName = "Input";
+
+        private static readonly string hostnamePlaceholder = "<HOSTNAME>";
+        private static readonly string rootPathPlaceholder = "/<ROOTPATH>";
+
+        public static string GetActualSubFolderPath(string testSubPath, string testName, string subFolderName)
+        {
+            return Path.Combine(TestHelper.GetActualOutputFolderPath(testSubPath, testName), subFolderName);
+        }
+
+        public static string GetExpectedFileContent(string testSubPath, string testName, string fileName)
+        {
+            return File.ReadAllText(Path.Combine(TestHelper.GetActualOutputFolderPath(testSubPath, testName), expectedFolderName, fileName));
+        }
+
+        public static void SaveModelJsonToActualOutput(string testSubPath, string testName, string fileName, string content)
+        {
+            string actualFolderPath = GetActualSubFolderPath(testSubPath, testName, actualFolderName);
+            if (!Directory.Exists(actualFolderPath))
+            {
+                Directory.CreateDirectory(actualFolderPath);
+            }
+
+            File.WriteAllText(
+                 Path.Combine(actualFolderPath, fileName),
+                 content);
+        }
+
+        public static void UpdateInputAndExpectedAndSaveToActualSubFolder(string testSubPath, string testName, string rootRelativePath, string fileName, string stringToAddSuffix, string suffix)
+        {
+            // Prepare the root path.
+            string rootPath = Environment.GetEnvironmentVariable("ADLS_ROOTPATH");
+            rootPath = AdlsTestHelper.GetFullRootPath(rootPath, rootRelativePath);
+
+            if (!rootPath.StartsWith("/"))
+            {
+                rootPath = $"/{rootPath}";
+            }
+
+            if (rootPath.EndsWith("/"))
+            {
+                rootPath = rootPath.Substring(0, rootPath.Length - 1);
+            }
+
+            rootPath = Uri.EscapeDataString(rootPath).Replace("%2F", "/");
+
+            // Update input content and save to ActualOutput/Input
+            string inputContent = TestHelper.GetInputFileContent(testSubPath, testName, fileName);
+            inputContent = ReplacePlaceholder(inputContent, rootPath, stringToAddSuffix, suffix);
+
+            string inputFolderPath = GetActualSubFolderPath(testSubPath, testName, inputFolderName);
+
+            if (!Directory.Exists(inputFolderPath))
+            {
+                Directory.CreateDirectory(inputFolderPath);
+            }
+
+            File.WriteAllText(
+                 Path.Combine(inputFolderPath, fileName),
+                 inputContent);
+
+            // Update expected content and save to ActualOutput/Expected
+            string expectedContent = TestHelper.GetExpectedOutputFileContent(testSubPath, testName, fileName);
+            expectedContent = ReplacePlaceholder(expectedContent, rootPath, stringToAddSuffix, suffix);
+
+            string expectedFolderPath = GetActualSubFolderPath(testSubPath, testName, expectedFolderName);
+
+            if (!Directory.Exists(expectedFolderPath))
+            {
+                Directory.CreateDirectory(expectedFolderPath);
+            }
+
+            File.WriteAllText(
+                 Path.Combine(expectedFolderPath, fileName),
+                 expectedContent);
+        }
+
+        private static string ReplacePlaceholder(string content, string root, string stringToAddSuffix, string suffix)
+        {
+            content = content.Replace(hostnamePlaceholder, Environment.GetEnvironmentVariable("ADLS_HOSTNAME"));
+            content = content.Replace(rootPathPlaceholder, root);
+            content = content.Replace(stringToAddSuffix, $"{stringToAddSuffix}-{suffix}");
+            return content;
+        }
+    }
+}

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/AdlsTestHelper.cs

@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using Microsoft.CommonDataModel.ObjectModel.Storage;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+
+namespace Microsoft.CommonDataModel.ObjectModel.Tests
+{
+    /// <summary>
+    /// Class to initialize adls tests adapters.
+    /// </summary>
+    public static class AdlsTestHelper
+    {
+        public static void CheckADLSEnvironment()
+        {
+            if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("ADLS_RUNTESTS")))
+            {
+                // this will cause tests to appear as "Skipped" in the final result
+                Assert.Inconclusive("ADLS environment not set up");
+            }
+        }
+
+        public static ADLSAdapter CreateAdapterWithSharedKey(string rootRelativePath = null)
+        {
+            string hostname = Environment.GetEnvironmentVariable("ADLS_HOSTNAME");
+            string rootPath = Environment.GetEnvironmentVariable("ADLS_ROOTPATH");
+            string sharedKey = Environment.GetEnvironmentVariable("ADLS_SHAREDKEY");
+
+            Assert.IsFalse(String.IsNullOrEmpty(hostname), "ADLS_ENDPOINT not set");
+            Assert.IsFalse(String.IsNullOrEmpty(rootPath), "ADLS_ROOTPATH not set");
+            Assert.IsFalse(String.IsNullOrEmpty(sharedKey), "ADLS_SHAREDKEY not set");
+
+            ADLSAdapter adapter = new ADLSAdapter(hostname, GetFullRootPath(rootPath, rootRelativePath), sharedKey);
+
+            return adapter;
+        }
+
+        public static ADLSAdapter CreateAdapterWithClientId(string rootRelativePath = null)
+        {
+            string hostname = Environment.GetEnvironmentVariable("ADLS_HOSTNAME");
+            string rootPath = Environment.GetEnvironmentVariable("ADLS_ROOTPATH");
+            string tenant = Environment.GetEnvironmentVariable("ADLS_TENANT");
+            string clientId = Environment.GetEnvironmentVariable("ADLS_CLIENTID");
+            string clientSecret = Environment.GetEnvironmentVariable("ADLS_CLIENTSECRET");
+
+            Assert.IsFalse(String.IsNullOrEmpty(hostname), "ADLS_ENDPOINT not set");
+            Assert.IsFalse(String.IsNullOrEmpty(rootPath), "ADLS_ROOTPATH not set");
+            Assert.IsFalse(String.IsNullOrEmpty(tenant), "ADLS_TENANT not set");
+            Assert.IsFalse(String.IsNullOrEmpty(clientId), "ADLS_CLIENTID not set");
+            Assert.IsFalse(String.IsNullOrEmpty(clientSecret), "ADLS_CLIENTSECRET not set");
+
+            ADLSAdapter adapter = new ADLSAdapter(hostname, GetFullRootPath(rootPath, rootRelativePath), tenant, clientId, clientSecret);
+
+            return adapter;
+        }
+
+        public static string GetFullRootPath(string rootPath, string rootRelativePath)
+        {
+            if (rootRelativePath == null)
+            {
+                return rootPath;
+            }
+
+            if (rootPath.EndsWith("/"))
+            {
+                rootPath = rootPath.Substring(0, rootPath.Length - 1);
+            }
+
+            if (rootRelativePath.StartsWith("/"))
+            {
+                rootRelativePath = rootRelativePath.Substring(1);
+            }
+
+            return $"{rootPath}/{rootRelativePath}";
+        }
+    }
+}

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/CorpusTests.cs

@@ -82,6 +82,44 @@ public async Task TestLazyLoadImports()
             await corpus.FetchObjectAsync<CdmDocumentDefinition>("local:/doc.cdm.json", null, resOpt);
         }
 
+        /// <summary>
+        /// Tests if a document that was fetched with lazy load and imported by another document is property indexed when needed.
+        /// </summary>
+        [TestMethod]
+        public async Task TestLazyLoadCreateResolvedEntity()
+        {
+            var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestLazyLoadCreateResolvedEntity");
+            corpus.SetEventCallback(new EventCallback
+            {
+                Invoke = (level, message) =>
+                {
+                    // no error should be logged.
+                    Assert.Fail(message);
+                }
+            }, CdmStatusLevel.Warning);
+
+            // load with deferred imports.
+            var resOpt = new ResolveOptions()
+            {
+                ImportsLoadStrategy = ImportsLoadStrategy.LazyLoad
+            };
+
+            // load entB which is imported by entA document.
+            var docB = await corpus.FetchObjectAsync<CdmDocumentDefinition>("local:/entB.cdm.json", null, resOpt);
+            var entA = await corpus.FetchObjectAsync<CdmEntityDefinition>("local:/entA.cdm.json/entA", null, resOpt);
+
+            Assert.IsNull(entA.InDocument.ImportPriorities);
+            Assert.IsNull(docB.ImportPriorities);
+
+            // CreateResolvedEntityAsync will force the entA document to be indexed.
+            var resEntA = await entA.CreateResolvedEntityAsync("resolved-EntA");
+
+            // in CreateResolvedEntityAsync the documents should be indexed.
+            Assert.IsNotNull(entA.InDocument.ImportPriorities);
+            Assert.IsNotNull(docB.ImportPriorities);
+            Assert.IsNotNull(resEntA.InDocument.ImportPriorities);
+        }
+
         /// <summary>
         /// Tests the FetchObjectAsync function with the imports load strategy set to load.
         /// </summary>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DocumentDefinitionTests.cs

@@ -83,6 +83,10 @@ public async Task TestDeeperCircularImportWithMoniker()
             await docA.IndexIfNeeded(new ResolveOptions(), true);
 
             Assert.AreEqual(4, docA.ImportPriorities.ImportPriority.Count);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docA], 0, false);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docB], 1, false);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docD], 2, false);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docC], 3, false);
 
             // reset the importsPriorities.
             MarkDocumentsToIndex(folder.Documents);
@@ -104,11 +108,52 @@ public async Task TestDeeperCircularImportWithMoniker()
         }
 
         /// <summary>
-        /// Test if monikered imports are not being added to the priorityList.
+        /// Test when A -> B -> C/M -> D.
+        /// Index docB first then docA. Make sure that C does not appear in docA priority list.
+        /// </summary>
+        [TestMethod]
+        public async Task TestReadingCachedImportPriority()
+        {
+            var corpus = TestHelper.GetLocalCorpus("", "");
+            var folder = corpus.Storage.FetchRootFolder("local");
+
+            var docA = new CdmDocumentDefinition(corpus.Ctx, "A.cdm.json");
+            folder.Documents.Add(docA);
+            docA.Imports.Add("B.cdm.json");
+
+            var docB = new CdmDocumentDefinition(corpus.Ctx, "B.cdm.json");
+            folder.Documents.Add(docB);
+            docB.Imports.Add("C.cdm.json", "moniker");
+
+            var docC = new CdmDocumentDefinition(corpus.Ctx, "C.cdm.json");
+            folder.Documents.Add(docC);
+            docC.Imports.Add("D.cdm.json");
+
+            var docD = new CdmDocumentDefinition(corpus.Ctx, "D.cdm.json");
+            folder.Documents.Add(docD);
+
+            // index docB first and check its import priorities.
+            await docB.IndexIfNeeded(new ResolveOptions(), true);
+
+            Assert.AreEqual(3, docB.ImportPriorities.ImportPriority.Count);
+            AssertImportInfo(docB.ImportPriorities.ImportPriority[docB], 0, false);
+            AssertImportInfo(docB.ImportPriorities.ImportPriority[docD], 1, false);
+            AssertImportInfo(docB.ImportPriorities.ImportPriority[docC], 2, true);
+
+            // now index docA, which should read docB's priority list from the cache.
+            await docA.IndexIfNeeded(new ResolveOptions(), true);
+            Assert.AreEqual(3, docA.ImportPriorities.ImportPriority.Count);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docA], 0, false);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docB], 1, false);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docD], 2, false);
+        }
+
+        /// <summary>
+        /// Test if monikered imports are added to the end of the priority list.
         /// A -> B/M -> C
         /// </summary>
         [TestMethod]
-        public async Task TestMonikeredImportIsNotAdded()
+        public async Task TestMonikeredImportIsAddedToEnd()
         {
             var corpus = TestHelper.GetLocalCorpus("", "");
             var folder = corpus.Storage.FetchRootFolder("local");
@@ -128,9 +173,14 @@ public async Task TestMonikeredImportIsNotAdded()
             await docB.IndexIfNeeded(new ResolveOptions(docB), true);
             await docA.IndexIfNeeded(new ResolveOptions(docA), true);
 
-            // should only contain docA and docC, docB should be excluded.
-            Assert.AreEqual(2, docA.ImportPriorities.ImportPriority.Count);
+            // should contain all three documents.
+            Assert.AreEqual(3, docA.ImportPriorities.ImportPriority.Count);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docA], 0, false);
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docC], 1, false);
+            // docB is monikered so it should appear at the end of the list.
+            AssertImportInfo(docA.ImportPriorities.ImportPriority[docB], 2, true);
 
+            // make sure that the has circular import is set to false.
             Assert.IsFalse(docA.ImportPriorities.hasCircularImport);
             Assert.IsFalse(docB.ImportPriorities.hasCircularImport);
             Assert.IsFalse(docC.ImportPriorities.hasCircularImport);
@@ -166,5 +216,17 @@ private void MarkDocumentsToIndex(CdmDocumentCollection documents)
                 document.ImportPriorities = null;
             }
         }
+
+        /// <summary>
+        /// Helper function to assert the ImportInfo class.
+        /// </summary>
+        /// <param name="importInfo"></param>
+        /// <param name="expectedPriority"></param>
+        /// <param name="expectedIsMoniker"></param>
+        private void AssertImportInfo(ImportInfo importInfo, int expectedPriority, bool expectedIsMoniker)
+        {
+            Assert.AreEqual(expectedPriority, importInfo.Priority);
+            Assert.AreEqual(expectedIsMoniker, importInfo.IsMoniker);
+        }
     }
 }