Microsoft CDM Release Version Update: 1.0.10

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.ALL.nuspec

@@ -5,7 +5,7 @@
   <metadata>
 
     <id>Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.All</id>
-    <version>1.0.9</version>
+    <version>1.0.10</version>
     <description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</description>
 
     <authors>Microsoft Corporation.</authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>netstandard2.0;net45;net462</TargetFrameworks>
-    <version>1.0.9</version>
+    <version>1.0.10</version>
     <Description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</Description>
 
     <Authors>Microsoft Corporation</Authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.nuspec

@@ -5,7 +5,7 @@
   <metadata>
 
     <id>Microsoft.CommonDataModel.ObjectModel.Adapter.Adls</id>
-    <version>1.0.9</version>
+    <version>1.0.10</version>
     <description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</description>
 
     <authors>Microsoft Corporation.</authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Storage/ADLSAdapter.cs

@@ -20,9 +20,13 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
     using Newtonsoft.Json.Linq;
     using System.Diagnostics;
     using Microsoft.CommonDataModel.ObjectModel.Utilities;
+    using System.ComponentModel;
+    using System.IO;
 
-    public class ADLSAdapter : NetworkAdapter, StorageAdapter
+    public class ADLSAdapter : NetworkAdapter
     {
+        private const double ADLSDefaultTimeout = 6000;
+
         private AuthenticationContext Context;
 
         /// <summary>
@@ -86,9 +90,6 @@ private set
         /// </summary>
         public TokenProvider TokenProvider { get; set; }
 
-        /// <inheritdoc />
-        public string LocationHint { get; set; }
-
         /// <summary>
         /// The map from corpus path to adapter path.
         /// </summary>
@@ -109,6 +110,11 @@ private set
         /// </summary>
         private string subPath = "";
 
+        /// <summary>
+        /// A cache for storing last modified times of file paths.
+        /// </summary>
+        private Dictionary<string, DateTimeOffset> fileModifiedTimeCache = new Dictionary<string, DateTimeOffset>();
+
         /// <summary>
         /// The predefined ADLS resource.
         /// </summary>
@@ -131,58 +137,56 @@ private set
 
         internal const string Type = "adls";
 
+        /// <summary>
+        /// The default constructor, a user has to apply JSON config after creating it this way.
+        /// </summary>
+        public ADLSAdapter()
+        {
+            this.httpClient = new CdmHttpClient();
+            this.Timeout = TimeSpan.FromMilliseconds(ADLSAdapter.ADLSDefaultTimeout);
+        }
+
         /// <summary>
         /// The ADLS constructor for clientId/secret authentication.
         /// </summary>
-        public ADLSAdapter(string hostname, string root, string tenant, string clientId, string secret)
+        public ADLSAdapter(string hostname, string root, string tenant, string clientId, string secret) : this()
         {
             this.Hostname = hostname;
             this.Root = root;
             this.Tenant = tenant;
             this.ClientId = clientId;
             this.Secret = secret;
             this.Context = new AuthenticationContext("https://login.windows.net/" + this.Tenant);
-            this.httpClient = new CdmHttpClient();
-        }
-
-        /// <summary>
-        /// The default constructor, a user has to apply JSON config after creating it this way.
-        /// </summary>
-        public ADLSAdapter()
-        {
-            this.httpClient = new CdmHttpClient();
         }
 
         /// <summary>
         /// The ADLS constructor for shared key authentication.
         /// </summary>
-        public ADLSAdapter(string hostname, string root, string sharedKey)
+        public ADLSAdapter(string hostname, string root, string sharedKey) : this()
         {
             this.Hostname = hostname;
             this.Root = root;
             this.SharedKey = sharedKey;
-            this.httpClient = new CdmHttpClient();
         }
 
         /// <summary>
         /// The ADLS constructor for user-defined token provider.
         /// </summary>
-        public ADLSAdapter(string hostname, string root, TokenProvider tokenProvider)
+        public ADLSAdapter(string hostname, string root, TokenProvider tokenProvider) : this()
         {
             this.Hostname = hostname;
             this.Root = root;
             this.TokenProvider = tokenProvider;
-            this.httpClient = new CdmHttpClient();
         }
 
         /// <inheritdoc />
-        public bool CanRead()
+        public override bool CanRead()
         {
             return true;
         }
 
         /// <inheritdoc />
-        public async Task<string> ReadAsync(string corpusPath)
+        public override async Task<string> ReadAsync(string corpusPath)
         {
             string url = this.CreateAdapterPath(corpusPath);
 
@@ -195,13 +199,13 @@ public async Task<string> ReadAsync(string corpusPath)
         }
 
         /// <inheritdoc />
-        public bool CanWrite()
+        public override bool CanWrite()
         {
             return true;
         }
 
         /// <inheritdoc />
-        public async Task WriteAsync(string corpusPath, string data)
+        public override async Task WriteAsync(string corpusPath, string data)
         {
             if (EnsurePath($"{this.Root}{corpusPath}") == false)
             {
@@ -224,7 +228,7 @@ public async Task WriteAsync(string corpusPath, string data)
         }
 
         /// <inheritdoc />
-        public string CreateAdapterPath(string corpusPath)
+        public override string CreateAdapterPath(string corpusPath)
         {
             var formattedCorpusPath = this.FormatCorpusPath(corpusPath);
             if (formattedCorpusPath == null)
@@ -235,15 +239,15 @@ public string CreateAdapterPath(string corpusPath)
             if (adapterPaths.ContainsKey(formattedCorpusPath))
             {
                 return adapterPaths[formattedCorpusPath];
-            } 
+            }
             else
             {
                 return $"https://{this.Hostname}{this.Root}{formattedCorpusPath}";
             }
         }
 
         /// <inheritdoc />
-        public string CreateCorpusPath(string adapterPath)
+        public override string CreateCorpusPath(string adapterPath)
         {
             if (!string.IsNullOrEmpty(adapterPath))
             {
@@ -272,40 +276,51 @@ public string CreateCorpusPath(string adapterPath)
             return null;
         }
 
-        /// <inheritdoc />
-        public void ClearCache()
+        public override void ClearCache()
         {
-            return;
+            this.fileModifiedTimeCache.Clear();
         }
 
         /// <inheritdoc />
-        public async Task<DateTimeOffset?> ComputeLastModifiedTimeAsync(string corpusPath)
+        public override async Task<DateTimeOffset?> ComputeLastModifiedTimeAsync(string corpusPath)
         {
-            var adapterPath = this.CreateAdapterPath(corpusPath);
+            if (this.IsCacheEnabled && fileModifiedTimeCache.TryGetValue(corpusPath, out DateTimeOffset time))
+            {
+                return time;
+            }
+            else
+            {
+                var adapterPath = this.CreateAdapterPath(corpusPath);
 
-            var httpRequest = await this.BuildRequest(adapterPath, HttpMethod.Head);
+                var httpRequest = await this.BuildRequest(adapterPath, HttpMethod.Head);
 
-            try
-            {
-                using (var cdmResponse = await base.ExecuteRequest(httpRequest))
+                try
                 {
-                    if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
+                    using (var cdmResponse = await base.ExecuteRequest(httpRequest))
                     {
-                        return cdmResponse.Content.Headers.LastModified;
+                        if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
+                        {
+                            var lastTime = cdmResponse.Content.Headers.LastModified;
+                            if (this.IsCacheEnabled && lastTime.HasValue)
+                            {
+                                this.fileModifiedTimeCache[corpusPath] = lastTime.Value;
+                            }
+                            return lastTime;
+                        }
                     }
                 }
+                catch (HttpRequestException ex)
+                {
+                    // We don't have standard logger here, so use one from system diagnostics
+                    Debug.WriteLine($"ADLS file not found, skipping last modified time calculation for it. Exception: {ex}");
+                }
+
+                return null;
             }
-            catch (HttpRequestException ex)
-            {
-                // We don't have standard logger here, so use one from system diagnostics
-                Debug.WriteLine($"ADLS file not found, skipping last modified time calculation for it. Exception: {ex}");
-            }
-
-            return null;
         }
 
         /// <inheritdoc />
-        public async Task<List<string>> FetchAllFilesAsync(string folderCorpusPath)
+        public override async Task<List<string>> FetchAllFilesAsync(string folderCorpusPath)
         {
             if (folderCorpusPath == null)
             {
@@ -346,7 +361,15 @@ public async Task<List<string>> FetchAllFilesAsync(string folderCorpusPath)
                         string nameWithoutSubPath = this.subPath.Length > 0 && name.ToString().StartsWith(this.subPath) ?
                             name.ToString().Substring(this.subPath.Length + 1) : name.ToString();
 
-                        result.Add(this.FormatCorpusPath(nameWithoutSubPath));
+                        string path = this.FormatCorpusPath(nameWithoutSubPath);
+                        result.Add(path);
+
+                        jObject.TryGetValue("lastModified", StringComparison.OrdinalIgnoreCase, out JToken lastModifiedTime);
+
+                        if (this.IsCacheEnabled && DateTimeOffset.TryParse(lastModifiedTime.ToString(), out DateTimeOffset offset))
+                        {
+                            fileModifiedTimeCache[path] = offset;
+                        }
                     }
                 }
 
@@ -586,7 +609,7 @@ private bool EnsurePath(string pathFor)
         }
 
         /// <inheritdoc />
-        public string FetchConfig()
+        public override string FetchConfig()
         {
             var resultConfig = new JObject
             {
@@ -620,7 +643,7 @@ public string FetchConfig()
         }
 
         /// <inheritdoc />
-        public void UpdateConfig(string config)
+        public override void UpdateConfig(string config)
         {
             if (config == null)
             {

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/CorpusTests.cs

@@ -55,5 +55,90 @@ public async Task TestComputeLastModifiedTimeAsync()
 
             await corpus.ComputeLastModifiedTimeAsync("local:/default.manifest.cdm.json");
         }
+
+        /// <summary>
+        /// Tests the FetchObjectAsync function with the StrictValidation off.
+        /// </summary>
+        [TestMethod]
+        public async Task TestStrictValidationOff()
+        {
+            var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestStrictValidation");
+            corpus.SetEventCallback(new EventCallback
+            {
+                Invoke = (level, message) =>
+                {
+                    // when the strict validation is disabled, there should be no reference validation.
+                    // no error should be logged.
+                    Assert.Fail(message);
+                }
+            }, CdmStatusLevel.Warning);
+
+            // load with deferred imports.
+            var resOpt = new ResolveOptions()
+            {
+                StrictValidation = false
+            };
+            await corpus.FetchObjectAsync<CdmDocumentDefinition>("local:/doc.cdm.json", null, resOpt);
+        }
+
+        /// <summary>
+        /// Tests the FetchObjectAsync function with the StrictValidation on.
+        /// </summary>
+        [TestMethod]
+        public async Task TestStrictValidationOn()
+        {
+            int errorCount = 0;
+            var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestStrictValidation");
+            corpus.SetEventCallback(new EventCallback
+            {
+                Invoke = (level, message) =>
+                {
+                    if (message.Contains("Unable to resolve the reference"))
+                    {
+                        errorCount++;
+                    }
+                    else
+                    {
+                        Assert.Fail(message);
+                    }
+
+                }
+            }, CdmStatusLevel.Error);
+
+            // load with strict validation.
+            var resOpt = new ResolveOptions()
+            {
+                StrictValidation = true
+            };
+            await corpus.FetchObjectAsync<CdmDocumentDefinition>("local:/doc.cdm.json", null, resOpt);
+            Assert.AreEqual(1, errorCount);
+
+            errorCount = 0;
+            corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestStrictValidation");
+            corpus.SetEventCallback(new EventCallback
+            {
+                Invoke = (level, message) =>
+                {
+                    if (level == CdmStatusLevel.Warning && message.Contains("Unable to resolve the reference"))
+                    {
+                        errorCount++;
+                    }
+                    else
+                    {
+                        Assert.Fail(message);
+                    }
+
+                }
+            }, CdmStatusLevel.Warning);
+
+            // load with strict validation and shallow validation.
+            resOpt = new ResolveOptions()
+            {
+                StrictValidation = true,
+                ShallowValidation = true
+            };
+            await corpus.FetchObjectAsync<CdmDocumentDefinition>("local:/doc.cdm.json", null, resOpt);
+            Assert.AreEqual(1, errorCount);
+        }
     }
 }