diff --git a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs index 9a69cc7152..27f987199b 100644 --- a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs +++ b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs @@ -185,7 +185,7 @@ public static IEnumerable LoadImagesFromDirectory(string folder) public static string DownloadImageSet(string imagesDownloadFolder) { string fileName = "flower_photos_tiny_set_for_unit_tests.zip"; - string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_tiny_set_for_unit_test.zip"; + string url = "https://mlpublicassets.blob.core.windows.net/assets/datasets/flower_photos_tiny_set_for_unit_tests.zip"; Download(url, imagesDownloadFolder, fileName).Wait(); UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); diff --git a/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs b/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs index a648ca303a..50d6ec30ea 100644 --- a/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs +++ b/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs @@ -4,8 +4,6 @@ using System; using System.IO; -using Microsoft.ML.Internal.Utilities; -using Microsoft.ML.Runtime; using Microsoft.ML.TestFrameworkCommon; namespace Microsoft.ML.PerformanceTests @@ -41,25 +39,20 @@ public static string GetBenchmarkDataPathAndEnsureData(string name, string path if (File.Exists(filePath)) return filePath; - var mlContext = new MLContext(1); - int timeout = 10 * 60 * 1000; - string url = $"benchmarks/{name}"; - var localPath = path == "" ? - Path.GetFullPath(DataDir) : - Path.GetFullPath(Path.Combine(DataDir, path)); - - using (var ch = (mlContext as IHostEnvironment).Start("Ensuring dataset files are present.")) + string url; + if (path == "") + { + url = $"https://raw.githubusercontent.com/dotnet/machinelearning/main/test/data/{name.Replace('\\', '/')}"; + } + else { - var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync( - mlContext, ch, url, name, localPath, timeout); - ensureModel.Wait(); - var errorResult = ResourceManagerUtils.GetErrorMessage(out var errorMessage, ensureModel.Result); - if (errorResult != null) - { - throw ch.Except($"{errorMessage}\n{name} could not be downloaded!"); - } + var blobName = Path.GetFileName(name); + if (blobName == "WikiDetoxAnnotated160kRows.tsv") + blobName = "wikiDetoxAnnotated160kRows.tsv"; + url = $"https://mlpublicassets.blob.core.windows.net/assets/benchmarks/{blobName}"; } + TestDownloadUtils.DownloadFile(url, filePath, TimeSpan.FromMinutes(10)); return filePath; } } diff --git a/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs b/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs index 1aedf19d69..b57b50f30b 100644 --- a/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs +++ b/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs @@ -139,7 +139,7 @@ public static string DownloadImageSet(string imagesDownloadFolder) //SINGLE SMALL FLOWERS IMAGESET (200 files) string fileName = "flower_photos_small_set.zip"; - string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip/"; + string url = "https://mlpublicassets.blob.core.windows.net/assets/datasets/flower_photos_small_set.zip"; Download(url, imagesDownloadFolder, fileName); UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); diff --git a/test/Microsoft.ML.TensorFlow.Tests/TensorflowTests.cs b/test/Microsoft.ML.TensorFlow.Tests/TensorflowTests.cs index d9d362d020..10c700069a 100644 --- a/test/Microsoft.ML.TensorFlow.Tests/TensorflowTests.cs +++ b/test/Microsoft.ML.TensorFlow.Tests/TensorflowTests.cs @@ -8,8 +8,6 @@ using System.IO.Compression; using System.Linq; using Microsoft.ML.Data; -using Microsoft.ML.Internal.Utilities; -using Microsoft.ML.Runtime; using Microsoft.ML.TensorFlow; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; @@ -61,7 +59,6 @@ public sealed class TensorFlowScenariosTests : BaseTestClass, IClassFixture + + annotations + + PreserveNewest @@ -13,4 +17,8 @@ + + + + diff --git a/test/Microsoft.ML.TestFrameworkCommon/TestDownloadUtils.cs b/test/Microsoft.ML.TestFrameworkCommon/TestDownloadUtils.cs new file mode 100644 index 0000000000..b5490a087b --- /dev/null +++ b/test/Microsoft.ML.TestFrameworkCommon/TestDownloadUtils.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.ML.TestFrameworkCommon +{ + public static class TestDownloadUtils + { + public static void DownloadFile(string url, string filePath, TimeSpan timeout) + { + var directory = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(directory)) + Directory.CreateDirectory(directory); + + string temporaryFilePath = $"{filePath}.{Guid.NewGuid():N}.download"; + using (var client = new HttpClient { Timeout = Timeout.InfiniteTimeSpan }) + { + RetryHelper.Execute(() => + { + try + { + if (File.Exists(filePath)) + return; + + using (var cancellationSource = new CancellationTokenSource(timeout)) + using (var response = client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationSource.Token).GetAwaiter().GetResult()) + { + response.EnsureSuccessStatusCode(); + long? expectedLength = response.Content.Headers.ContentLength; + using (var contentStream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult()) + using (var fileStream = new FileStream(temporaryFilePath, FileMode.Create, FileAccess.Write, FileShare.None)) + { + contentStream.CopyToAsync(fileStream, 81920, cancellationSource.Token).GetAwaiter().GetResult(); + if (expectedLength.HasValue && fileStream.Length != expectedLength.Value) + throw new IOException($"Expected {expectedLength.Value} bytes from '{url}', but downloaded {fileStream.Length} bytes."); + } + } + + try + { + File.Move(temporaryFilePath, filePath); + } + catch (IOException) when (File.Exists(filePath)) + { + // Another caller completed the same download first. + } + } + finally + { + if (File.Exists(temporaryFilePath)) + File.Delete(temporaryFilePath); + } + }, backoffFunc: attempt => 10_000, + retryWhen: ex => ex is HttpRequestException || ex is TaskCanceledException || ex is IOException); + } + } + } +}