Avoid reading the whole file into memory when adding a file to a .zip archive

Files.Package/Files.Package.wapproj

@@ -477,5 +477,9 @@
     <Content Include="Assets\AppTiles\Wide310x150Logo.scale-200.png" />
     <Content Include="Assets\AppTiles\Wide310x150Logo.scale-400.png" />
   </ItemGroup>
+  <ItemGroup>
+    <None Include="nupkgs\microsoft.management.infrastructure.runtime.win.2.0.1.nupkg" />
+    <None Include="nupkgs\SharpZipLib.1.3.4.nupkg" />
+  </ItemGroup>
   <Import Project="$(WapProjPath)\Microsoft.DesktopBridge.targets" />
 </Project>

Files/Files.csproj

@@ -1466,17 +1466,14 @@
     <PackageReference Include="Microsoft.UI.Xaml">
       <Version>2.7.0</Version>
     </PackageReference>
-    <PackageReference Include="Microsoft.Win32.Registry">
-      <Version>5.0.0</Version>
-    </PackageReference>
     <PackageReference Include="Microsoft.Xaml.Behaviors.Uwp.Managed">
       <Version>2.0.1</Version>
     </PackageReference>
     <PackageReference Include="Newtonsoft.Json">
       <Version>13.0.1</Version>
     </PackageReference>
     <PackageReference Include="SharpZipLib">
-      <Version>1.3.3</Version>
+      <Version>1.3.4</Version>
     </PackageReference>
     <PackageReference Include="SQLitePCLRaw.bundle_green">
       <Version>2.0.7</Version>

Files/Filesystem/StorageItems/FtpStorageFile.cs

@@ -178,31 +178,21 @@ public override IAsyncOperation<IRandomAccessStream> OpenAsync(FileAccessMode ac
                 if (accessMode == FileAccessMode.Read)
                 {
                     var inStream = await ftpClient.OpenReadAsync(FtpPath, cancellationToken);
-                    return new NonSeekableRandomAccessStream(inStream, (ulong)inStream.Length)
+                    return new NonSeekableRandomAccessStreamForRead(inStream, (ulong)inStream.Length)
                     {
                         DisposeCallback = () => ftpClient.Dispose()
                     };
                 }
                 else
                 {
-                    return new RandomAccessStreamWithFlushCallback()
+                    return new NonSeekableRandomAccessStreamForWrite(await ftpClient.OpenWriteAsync(FtpPath, cancellationToken))
                     {
-                        DisposeCallback = () => ftpClient.Dispose(),
-                        FlushCallback = UploadFile(ftpClient)
+                        DisposeCallback = () => ftpClient.Dispose()
                     };
                 }
             });
         }
 
-        private Func<IRandomAccessStream, IAsyncOperation<bool>> UploadFile(FtpClient ftpClient)
-        {
-            return (stream) => AsyncInfo.Run(async (cancellationToken) =>
-            {
-                await ftpClient.UploadAsync(stream.CloneStream().AsStream(), FtpPath, FtpRemoteExists.Overwrite);
-                return true;
-            });
-        }
-
         public override IAsyncOperation<StorageStreamTransaction> OpenTransactedWriteAsync() => throw new NotSupportedException();
 
         public override IAsyncOperation<BaseStorageFile> CopyAsync(IStorageFolder destinationFolder)
@@ -267,7 +257,7 @@ public override IAsyncOperation<IRandomAccessStreamWithContentType> OpenReadAsyn
                 }
 
                 var inStream = await ftpClient.OpenReadAsync(FtpPath, cancellationToken);
-                var nsStream = new NonSeekableRandomAccessStream(inStream, (ulong)inStream.Length)
+                var nsStream = new NonSeekableRandomAccessStreamForRead(inStream, (ulong)inStream.Length)
                 {
                     DisposeCallback = () => ftpClient.Dispose()
                 };

Files/Filesystem/StorageItems/StreamWithContentType.cs

@@ -10,74 +10,98 @@ namespace Files.Filesystem.StorageItems
 {
     public class InputStreamWithDisposeCallback : IInputStream
     {
-        private IInputStream stream;
+        private Stream stream;
+        private IInputStream iStream;
         public Action DisposeCallback { get; set; }
 
         public InputStreamWithDisposeCallback(Stream stream)
         {
-            this.stream = stream.AsInputStream();
+            this.stream = stream;
+            this.iStream = stream.AsInputStream();
         }
 
         public IAsyncOperationWithProgress<IBuffer, uint> ReadAsync(IBuffer buffer, uint count, InputStreamOptions options)
         {
-            return stream.ReadAsync(buffer, count, options);
+            return iStream.ReadAsync(buffer, count, options);
         }
 
         public void Dispose()
         {
+            iStream.Dispose();
             stream.Dispose();
             DisposeCallback?.Invoke();
         }
     }
 
-    public class RandomAccessStreamWithFlushCallback : IRandomAccessStream
+    public class NonSeekableRandomAccessStreamForWrite : IRandomAccessStream
     {
+        private Stream stream;
+        private IOutputStream oStream;
         private IRandomAccessStream imrac;
+        private ulong byteSize;
         private bool isWritten;
-        public Func<IRandomAccessStream, IAsyncOperation<bool>> FlushCallback { get; set; }
+
         public Action DisposeCallback { get; set; }
 
-        public RandomAccessStreamWithFlushCallback()
+        public NonSeekableRandomAccessStreamForWrite(Stream stream)
         {
+            this.stream = stream;
+            this.oStream = stream.AsOutputStream();
             this.imrac = new InMemoryRandomAccessStream();
         }
 
         public IInputStream GetInputStreamAt(ulong position)
         {
-            return imrac.GetInputStreamAt(position);
+            throw new NotSupportedException();
         }
 
         public IOutputStream GetOutputStreamAt(ulong position)
         {
-            return imrac.GetOutputStreamAt(position);
+            if (position != 0)
+            {
+                throw new NotSupportedException();
+            }
+            return this;
         }
 
         public void Seek(ulong position)
         {
-            imrac.Seek(position);
+            if (position != 0)
+            {
+                throw new NotSupportedException();
+            }
         }
 
-        public IRandomAccessStream CloneStream()
-        {
-            return imrac.CloneStream();
-        }
+        public IRandomAccessStream CloneStream() => throw new NotSupportedException();
 
-        public bool CanRead => imrac.CanRead;
+        public bool CanRead => false;
 
-        public bool CanWrite => imrac.CanWrite;
+        public bool CanWrite => true;
 
-        public ulong Position => imrac.Position;
+        public ulong Position => byteSize;
 
-        public ulong Size { get => imrac.Size; set => imrac.Size = value; }
+        public ulong Size
+        {
+            get => byteSize;
+            set => throw new NotSupportedException();
+        }
 
         public IAsyncOperationWithProgress<IBuffer, uint> ReadAsync(IBuffer buffer, uint count, InputStreamOptions options)
         {
-            return imrac.ReadAsync(buffer, count, options);
+            throw new NotSupportedException();
         }
 
         public IAsyncOperationWithProgress<uint, uint> WriteAsync(IBuffer buffer)
         {
-            return imrac.WriteAsync(buffer);
+            Func<CancellationToken, IProgress<uint>, Task<uint>> taskProvider =
+                async (token, progress) =>
+                {
+                    var res = await oStream.WriteAsync(buffer);
+                    byteSize += res;
+                    return res;
+                };
+
+            return AsyncInfo.Run(taskProvider);
         }
 
         public IAsyncOperation<bool> FlushAsync()
@@ -89,17 +113,23 @@ public IAsyncOperation<bool> FlushAsync()
 
             isWritten = true;
 
-            return FlushCallback(this) ?? imrac.FlushAsync();
+            return AsyncInfo.Run<bool>(async (cancellationToken) =>
+            {
+                await stream.FlushAsync();
+                return true;
+            });
         }
 
         public void Dispose()
         {
+            oStream.Dispose();
+            stream.Dispose();
             imrac.Dispose();
             DisposeCallback?.Invoke();
         }
     }
 
-    public class NonSeekableRandomAccessStream : IRandomAccessStream
+    public class NonSeekableRandomAccessStreamForRead : IRandomAccessStream
     {
         private Stream stream;
         private IRandomAccessStream imrac;
@@ -109,7 +139,7 @@ public class NonSeekableRandomAccessStream : IRandomAccessStream
 
         public Action DisposeCallback { get; set; }
 
-        public NonSeekableRandomAccessStream(Stream baseStream, ulong size)
+        public NonSeekableRandomAccessStreamForRead(Stream baseStream, ulong size)
         {
             this.stream = baseStream;
             this.imrac = new InMemoryRandomAccessStream();
@@ -135,9 +165,9 @@ public void Seek(ulong position)
             this.virtualPosition = position;
         }
 
-        public IRandomAccessStream CloneStream() => imrac.CloneStream();
+        public IRandomAccessStream CloneStream() => throw new NotSupportedException();
 
-        public bool CanRead => imrac.CanRead;
+        public bool CanRead => true;
 
         public bool CanWrite => false;
 

Files/Filesystem/StorageItems/ZipStorageFile.cs

@@ -56,31 +56,57 @@ public override IAsyncOperation<IRandomAccessStream> OpenAsync(FileAccessMode ac
                     }
                 }
 
-                ZipFile zipFile = await OpenZipFileAsync(accessMode);
-                if (zipFile == null)
-                {
-                    return null;
-                }
-                zipFile.IsStreamOwner = true;
-                var znt = new ZipNameTransform(ContainerPath);
-                var entry = zipFile.GetEntry(znt.TransformFile(Path));
                 if (!rw)
                 {
+                    ZipFile zipFile = await OpenZipFileAsync(accessMode);
+                    if (zipFile == null)
+                    {
+                        return null;
+                    }
+                    zipFile.IsStreamOwner = true;
+                    var znt = new ZipNameTransform(ContainerPath);
+                    var entry = zipFile.GetEntry(znt.TransformFile(Path));
                     if (entry != null)
                     {
-                        return new NonSeekableRandomAccessStream(zipFile.GetInputStream(entry), (ulong)entry.Size)
+                        return new NonSeekableRandomAccessStreamForRead(zipFile.GetInputStream(entry), (ulong)entry.Size)
                         {
                             DisposeCallback = () => zipFile.Close()
                         };
                     }
                 }
                 else
                 {
-                    return new RandomAccessStreamWithFlushCallback()
+                    var znt = new ZipNameTransform(ContainerPath);
+                    var zipDesiredName = znt.TransformFile(Path);
+
+                    using (ZipFile zipFile = await OpenZipFileAsync(accessMode))
                     {
-                        DisposeCallback = () => zipFile.Close(),
-                        FlushCallback = WriteZipEntry(zipFile)
-                    };
+                        var entry = zipFile.GetEntry(zipDesiredName);
+                        if (entry != null)
+                        {
+                            zipFile.BeginUpdate(new MemoryArchiveStorage(FileUpdateMode.Direct));
+                            zipFile.Delete(entry);
+                            zipFile.CommitUpdate();
+                        }
+                    }
+
+                    if (BackingFile != null)
+                    {
+                        var zos = new ZipOutputStream((await BackingFile.OpenAsync(FileAccessMode.ReadWrite)).AsStream(), true);
+                        await zos.PutNextEntryAsync(new ZipEntry(zipDesiredName));
+                        return new NonSeekableRandomAccessStreamForWrite(zos);
+                    }
+                    else
+                    {
+                        var hFile = NativeFileOperationsHelper.OpenFileForRead(ContainerPath, true);
+                        if (hFile.IsInvalid)
+                        {
+                            return null;
+                        }
+                        var zos = new ZipOutputStream(new FileStream(hFile, FileAccess.ReadWrite), true);
+                        await zos.PutNextEntryAsync(new ZipEntry(zipDesiredName));
+                        return new NonSeekableRandomAccessStreamForWrite(zos);
+                    }
                 }
                 return null;
             });
@@ -249,7 +275,7 @@ public override IAsyncOperation<IRandomAccessStreamWithContentType> OpenReadAsyn
                 var entry = zipFile.GetEntry(znt.TransformFile(Path));
                 if (entry != null)
                 {
-                    var nsStream = new NonSeekableRandomAccessStream(zipFile.GetInputStream(entry), (ulong)entry.Size)
+                    var nsStream = new NonSeekableRandomAccessStreamForRead(zipFile.GetInputStream(entry), (ulong)entry.Size)
                     {
                         DisposeCallback = () => zipFile.Close()
                     };
@@ -382,7 +408,7 @@ await NativeFileOperationsHelper.OpenProtectedFileForRead(ContainerPath) :
                     {
                         return null;
                     }
-                    return new ZipFile(new FileStream(hFile, readWrite ? FileAccess.ReadWrite : FileAccess.Read));
+                    return new ZipFile((Stream)new FileStream(hFile, readWrite ? FileAccess.ReadWrite : FileAccess.Read));
                 }
             });
         }
@@ -428,32 +454,6 @@ private StreamedFileDataRequestedHandler ZipDataStreamingHandler(string name)
             };
         }
 
-        private Func<IRandomAccessStream, IAsyncOperation<bool>> WriteZipEntry(ZipFile zipFile)
-        {
-            return (stream) => AsyncInfo.Run((cancellationToken) => Task.Run(() =>
-            {
-                try
-                {
-                    var znt = new ZipNameTransform(ContainerPath);
-                    var zipDesiredName = znt.TransformFile(Path);
-                    var entry = zipFile.GetEntry(zipDesiredName);
-
-                    zipFile.BeginUpdate(new MemoryArchiveStorage(FileUpdateMode.Direct));
-                    if (entry != null)
-                    {
-                        zipFile.Delete(entry);
-                    }
-                    zipFile.Add(new StreamDataSource(stream), zipDesiredName);
-                    zipFile.CommitUpdate();
-                }
-                catch (Exception ex)
-                {
-                    App.Logger.Warn(ex, "Error writing zip file");
-                }
-                return true;
-            }));
-        }
-
         private async Task<BaseBasicProperties> GetBasicProperties()
         {
             using (ZipFile zipFile = await OpenZipFileAsync(FileAccessMode.Read))
@@ -510,18 +510,6 @@ public ZipFileBasicProperties(ZipEntry entry)
             public override ulong Size => (ulong)zipEntry.Size;
         }
 
-        private class StreamDataSource : IStaticDataSource
-        {
-            private IRandomAccessStream stream;
-
-            public StreamDataSource(IRandomAccessStream stream)
-            {
-                this.stream = stream;
-            }
-
-            public Stream GetSource() => stream.CloneStream().AsStream();
-        }
-
         #endregion
     }
 }

Files/Filesystem/StorageItems/ZipStorageFolder.cs

@@ -580,7 +580,7 @@ private IAsyncOperation<ZipFile> OpenZipFileAsync(FileAccessMode accessMode)
                     {
                         return null;
                     }
-                    return new ZipFile(new FileStream(hFile, readWrite ? FileAccess.ReadWrite : FileAccess.Read));
+                    return new ZipFile((Stream)new FileStream(hFile, readWrite ? FileAccess.ReadWrite : FileAccess.Read));
                 }
             });
         }

nuget.config

@@ -1,7 +1,7 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
     <packageSources>
         <add key="NuGet official package source" value="https://api.nuget.org/v3/index.json" />
-        <add key="Local Packages" value="Files.Launcher\nupkgs\" />
+        <add key="Local Packages" value="Files.Package\nupkgs\" />
     </packageSources>
 </configuration>