diff --git a/Source/Tools/Flax.Build/Deps/Downloader.cs b/Source/Tools/Flax.Build/Deps/Downloader.cs index 08f445dd1..e1e80a2f0 100644 --- a/Source/Tools/Flax.Build/Deps/Downloader.cs +++ b/Source/Tools/Flax.Build/Deps/Downloader.cs @@ -1,11 +1,6 @@ -// Copyright (c) 2012-2022 Wojciech Figat. All rights reserved. +// Copyright (c) 2012-2022 Wojciech Figat. All rights reserved. -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.IO; using System.Net; -using System.Threading; using Flax.Build; namespace Flax.Deps @@ -15,133 +10,52 @@ namespace Flax.Deps /// static class Downloader { - /* - private static void DownloadFile(string fileUrl, string fileTmp) - { - HttpWebRequest webRequest = WebRequest.CreateHttp(fileUrl); - webRequest.Method = "GET"; - webRequest.Timeout = 3000; - using (WebResponse webResponse = webRequest.GetResponse()) - { - using (Stream remoteStream = webResponse.GetResponseStream()) - { - DownloadFile(remoteStream, webResponse.ContentLength, fileTmp); - } - } - } - - private static void DownloadFile(Stream remoteStream, long length, string fileTmp) - { - long bytesProcessed = 0; - byte[] buffer = new byte[1024]; - - using (var localStream = new FileStream(fileTmp, FileMode.Create, FileAccess.Write, FileShare.Read)) - { - using (var display = new ProgressDisplay(length)) - { - int bytesRead; - do - { - bytesRead = remoteStream.Read(buffer, 0, buffer.Length); - localStream.Write(buffer, 0, bytesRead); - bytesProcessed += bytesRead; - display.Progress = bytesProcessed; - } while (bytesRead > 0); - } - } - } - */ - private const string GoogleDriveDomain = "drive.google.com"; private const string GoogleDriveDomain2 = "https://drive.google.com"; // Normal example: FileDownloader.DownloadFileFromURLToPath( "http://example.com/file/download/link", @"C:\file.txt" ); // Drive example: FileDownloader.DownloadFileFromURLToPath( "http://drive.google.com/file/d/FILEID/view?usp=sharing", @"C:\file.txt" ); - public static FileInfo DownloadFileFromUrlToPath(string url, string path) { Log.Verbose(string.Format("Downloading {0} to {1}", url, path)); if (File.Exists(path)) File.Delete(path); - if (url.StartsWith(GoogleDriveDomain) || url.StartsWith(GoogleDriveDomain2)) return DownloadGoogleDriveFileFromUrlToPath(url, path); return DownloadFileFromUrlToPath(url, path, null); } - private static FileInfo DownloadFileFromUrlToPathRaw(string url, string path, WebClient webClient) + private static FileInfo DownloadFileFromUrlToPathRaw(string url, string path, HttpClient httpClient) { if (ProgressDisplay.CanUseConsole) { - using (var display = new ProgressDisplay(0)) + using (var progress = new ProgressDisplay(0)) { - DownloadProgressChangedEventHandler downloadProgress = (sender, e) => { display.Update(e.BytesReceived, e.TotalBytesToReceive); }; - AsyncCompletedEventHandler downloadCompleted = (sender, e) => - { - lock (e.UserState) - { - Monitor.Pulse(e.UserState); - } - }; - - var syncObj = new object(); - lock (syncObj) - { - webClient.DownloadProgressChanged += downloadProgress; - webClient.DownloadFileCompleted += downloadCompleted; - - webClient.DownloadFileAsync(new Uri(url), path, syncObj); - Monitor.Wait(syncObj); - - webClient.DownloadProgressChanged -= downloadProgress; - webClient.DownloadFileCompleted -= downloadCompleted; - } + var task = DownloadFileFromUrlToPathAsync(url, path, httpClient, progress); + task.Wait(); } } else { - AsyncCompletedEventHandler downloadCompleted = (sender, e) => - { - lock (e.UserState) - { - if (e.Error != null) - { - Log.Error("Download failed."); - Log.Exception(e.Error); - } - - Monitor.Pulse(e.UserState); - } - }; - - var syncObj = new object(); - lock (syncObj) - { - webClient.DownloadFileCompleted += downloadCompleted; - - webClient.DownloadFileAsync(new Uri(url), path, syncObj); - Monitor.Wait(syncObj); - - webClient.DownloadFileCompleted -= downloadCompleted; - } + var task = DownloadFileFromUrlToPathAsync(url, path, httpClient); + task.Wait(); } return new FileInfo(path); } - private static FileInfo DownloadFileFromUrlToPath(string url, string path, WebClient webClient) + private static FileInfo DownloadFileFromUrlToPath(string url, string path, HttpClient httpClient) { try { - if (webClient == null) + if (httpClient == null) { - using (webClient = new WebClient()) + using (httpClient = new HttpClient()) { - return DownloadFileFromUrlToPathRaw(url, path, webClient); + return DownloadFileFromUrlToPathRaw(url, path, httpClient); } } - - return DownloadFileFromUrlToPathRaw(url, path, webClient); + return DownloadFileFromUrlToPathRaw(url, path, httpClient); } catch (WebException) { @@ -149,6 +63,62 @@ namespace Flax.Deps } } + private static async Task DownloadFileFromUrlToPathAsync(string url, string path, HttpClient httpClient, ProgressDisplay progress = null) + { + if (progress != null) + { + using (var response = await httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead)) + { + response.EnsureSuccessStatusCode(); + + var totalBytes = response.Content.Headers.ContentLength; + using (var contentStream = await response.Content.ReadAsStreamAsync()) + { + var totalBytesRead = 0L; + var readCount = 0L; + var buffer = new byte[8192]; + var hasMoreToRead = true; + using (var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None)) + { + do + { + var bytesRead = await contentStream.ReadAsync(buffer, 0, buffer.Length); + if (bytesRead == 0) + { + hasMoreToRead = false; + if (totalBytes.HasValue) + progress.Update(totalBytesRead, totalBytes.Value); + continue; + } + + await fileStream.WriteAsync(buffer, 0, bytesRead); + + totalBytesRead += bytesRead; + readCount += 1; + + if (readCount % 10 == 0) + { + if (totalBytes.HasValue) + progress.Update(totalBytesRead, totalBytes.Value); + } + } + while (hasMoreToRead); + } + } + } + } + else + { + using (var httpStream = await httpClient.GetStreamAsync(url)) + { + using (var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None)) + { + await httpStream.CopyToAsync(fileStream); + } + } + } + } + // Downloading large files from Google Drive prompts a warning screen and // requires manual confirmation. Consider that case and try to confirm the download automatically // if warning prompt occurs @@ -157,8 +127,7 @@ namespace Flax.Deps // You can comment the statement below if the provided url is guaranteed to be in the following format: // https://drive.google.com/uc?id=FILEID&export=download url = GetGoogleDriveDownloadLinkFromUrl(url); - - using (CookieAwareWebClient webClient = new CookieAwareWebClient()) + using (var httpClient = new HttpClient()) { FileInfo downloadedFile; @@ -166,7 +135,7 @@ namespace Flax.Deps // but works in the second attempt for (int i = 0; i < 2; i++) { - downloadedFile = DownloadFileFromUrlToPath(url, path, webClient); + downloadedFile = DownloadFileFromUrlToPath(url, path, httpClient); if (downloadedFile == null) return null; @@ -199,8 +168,7 @@ namespace Flax.Deps url = "https://drive.google.com" + content.Substring(linkIndex, linkEnd - linkIndex).Replace("&", "&"); } - downloadedFile = DownloadFileFromUrlToPath(url, path, webClient); - + downloadedFile = DownloadFileFromUrlToPath(url, path, httpClient); return downloadedFile; } } @@ -225,7 +193,6 @@ namespace Flax.Deps index = url.IndexOf("file/d/", StringComparison.Ordinal); if (index < 0) // url is not in any of the supported forms return string.Empty; - index += 7; closingIndex = url.IndexOf('/', index); @@ -240,127 +207,4 @@ namespace Flax.Deps return string.Format("https://drive.google.com/uc?id={0}&export=download", url.Substring(index, closingIndex - index)); } } - - /// - /// A with exposed Response property. - /// - /// - public class WebClientWithResponse : WebClient - { - /// - /// Gets the response. - /// - public byte[] Response { get; private set; } - - /// - protected override WebResponse GetWebResponse(WebRequest request) - { - var response = base.GetWebResponse(request); - - if (response is HttpWebResponse httpResponse) - { - using (var stream = httpResponse.GetResponseStream()) - { - using (var ms = new MemoryStream()) - { - stream.CopyTo(ms); - Response = ms.ToArray(); - } - } - } - - return response; - } - } - - /// - /// Web client used for Google Drive - /// - /// - public class CookieAwareWebClient : WebClient - { - private class CookieContainer - { - private readonly Dictionary _cookies; - - public string this[Uri url] - { - get - { - string cookie; - if (_cookies.TryGetValue(url.Host, out cookie)) - return cookie; - - return null; - } - set { _cookies[url.Host] = value; } - } - - public CookieContainer() - { - _cookies = new Dictionary(); - } - } - - private readonly CookieContainer _cookies; - - /// - /// Initializes a new instance of the class. - /// - public CookieAwareWebClient() - { - _cookies = new CookieContainer(); - } - - /// - protected override WebRequest GetWebRequest(Uri address) - { - WebRequest request = base.GetWebRequest(address); - - if (request is HttpWebRequest) - { - string cookie = _cookies[address]; - if (cookie != null) - ((HttpWebRequest)request).Headers.Set("cookie", cookie); - } - - return request; - } - - /// - protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result) - { - WebResponse response = base.GetWebResponse(request, result); - - string[] cookies = response.Headers.GetValues("Set-Cookie"); - if (cookies != null && cookies.Length > 0) - { - string cookie = ""; - foreach (string c in cookies) - cookie += c; - - _cookies[response.ResponseUri] = cookie; - } - - return response; - } - - /// - protected override WebResponse GetWebResponse(WebRequest request) - { - WebResponse response = base.GetWebResponse(request); - - string[] cookies = response.Headers.GetValues("Set-Cookie"); - if (cookies != null && cookies.Length > 0) - { - string cookie = ""; - foreach (string c in cookies) - cookie += c; - - _cookies[response.ResponseUri] = cookie; - } - - return response; - } - } } diff --git a/Source/Tools/Flax.Build/Flax.Build.csproj b/Source/Tools/Flax.Build/Flax.Build.csproj index 6227c8c38..3aeb41ddc 100644 --- a/Source/Tools/Flax.Build/Flax.Build.csproj +++ b/Source/Tools/Flax.Build/Flax.Build.csproj @@ -1,4 +1,4 @@ - + Exe net7.0 @@ -23,7 +23,7 @@ - +