Vulkan support

LLama/LLamaSharp.Runtime.targets

@@ -52,6 +52,14 @@
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/win-x64/native/cuda12/ggml.dll</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/llama.dll">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/vulkan/llama.dll</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/ggml.dll">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/vulkan/ggml.dll</Link>
+      </None>
 
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/libllama.so">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -101,6 +109,14 @@
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/linux-x64/native/cuda12/libggml.so</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libllama.so">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-x64/native/vulkan/libllama.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libggml.so">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-x64/native/vulkan/libggml.so</Link>
+      </None>
 
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libggml.dylib">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -169,6 +185,10 @@
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/win-x64/native/cuda12/llava_shared.dll</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/llava_shared.dll">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/vulkan/llava_shared.dll</Link>
+      </None>
 
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/libllava_shared.so">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -194,5 +214,9 @@
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/linux-x64/native/cuda12/libllava_shared.so</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libllava_shared.so">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-x64/native/vulkan/libllava_shared.so</Link>
+      </None>
     </ItemGroup>
 </Project>

LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs

@@ -28,7 +28,12 @@ public IEnumerable<INativeLibrary> Apply(NativeLibraryConfig.Description descrip
                     yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck);
                 }
 
-                if(!description.UseCuda || description.AllowFallback)
+                if (description.UseVulkan)
+                {
+                    yield return new NativeLibraryWithVulkan(systemInfo.VulkanVersion, description.Library, description.SkipCheck);
+                }
+
+                if((!description.UseCuda || !description.UseVulkan) || description.AllowFallback)
                 {
                     if (description.AllowFallback)
                     {

LLama/Native/Load/NativeLibraryConfig.cs

@@ -16,6 +16,7 @@ public sealed partial class NativeLibraryConfig
         private string? _libraryPath;
 
         private bool _useCuda = true;
+        private bool _useVulkan = true;
         private AvxLevel _avxLevel;
         private bool _allowFallback = true;
         private bool _skipCheck = false;
@@ -55,6 +56,20 @@ public NativeLibraryConfig WithCuda(bool enable = true)
             _useCuda = enable;
             return this;
         }
+
+        /// <summary>
+        /// Configure whether to use vulkan backend if possible. Default is true.
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfig WithVulkan(bool enable = true)
+        {
+            ThrowIfLoaded();
+
+            _useVulkan = enable;
+            return this;
+        }
 
         /// <summary>
         /// Configure the prefferred avx support level of the backend. 
@@ -159,6 +174,7 @@ internal Description CheckAndGatherDescription()
                 path,
                 NativeLibraryName,
                 _useCuda,
+                _useVulkan,
                 _avxLevel,
                 _allowFallback,
                 _skipCheck,
@@ -229,7 +245,7 @@ private static bool CheckAVX512()
         /// <param name="AllowFallback"></param>
         /// <param name="SkipCheck"></param>
         /// <param name="SearchDirectories"></param>
-        public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, 
+        public record Description(string? Path, NativeLibraryName Library, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, 
             string[] SearchDirectories)
         {
             /// <inheritdoc/>
@@ -250,6 +266,7 @@ public override string ToString()
                        $"- LibraryName: {Library}\n" +
                        $"- Path: '{Path}'\n" +
                        $"- PreferCuda: {UseCuda}\n" +
+                       $"- PreferVulkan: {UseVulkan}\n" +
                        $"- PreferredAvxLevel: {avxLevelString}\n" +
                        $"- AllowFallback: {AllowFallback}\n" +
                        $"- SkipCheck: {SkipCheck}\n" +
@@ -425,6 +442,21 @@ public NativeLibraryConfigContainer WithCuda(bool enable = true)
             }
             return this;
         }
+
+        /// <summary>
+        /// Configure whether to use vulkan backend if possible.
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfigContainer WithVulkan(bool enable = true)
+        {
+            foreach(var config in _configs)
+            {
+                config.WithVulkan(enable);
+            }
+            return this;
+        }
 
         /// <summary>
         /// Configure the prefferred avx support level of the backend.

LLama/Native/Load/NativeLibraryMetadata.cs

@@ -6,12 +6,13 @@ namespace LLama.Native
     /// </summary>
     /// <param name="NativeLibraryName">Which kind of library it is.</param>
     /// <param name="UseCuda">Whether it's compiled with cublas.</param>
+    /// <param name="UseVulkan">Whether it's compiled with vulkan.</param>
     /// <param name="AvxLevel">Which AvxLevel it's compiled with.</param>
-    public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, AvxLevel AvxLevel)
+    public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel)
     {
         public override string ToString()
         {
-            return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, AvxLevel: {AvxLevel})";
+            return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, UseVulkan: {UseVulkan}, AvxLevel: {AvxLevel})";
         }
     }
 

LLama/Native/Load/NativeLibraryWithAvx.cs

@@ -19,7 +19,7 @@ public NativeLibraryMetadata? Metadata
         {
             get
             {
-                return new NativeLibraryMetadata(_libraryName, false, _avxLevel);
+                return new NativeLibraryMetadata(_libraryName, false,  false,_avxLevel);
             }
         }
 

LLama/Native/Load/NativeLibraryWithCuda.cs

@@ -20,7 +20,7 @@ public NativeLibraryMetadata? Metadata
         {
             get
             {
-                return new NativeLibraryMetadata(_libraryName, true, _avxLevel);
+                return new NativeLibraryMetadata(_libraryName, true, false, _avxLevel);
             }
         }
 

LLama/Native/Load/NativeLibraryWithMacOrFallback.cs

@@ -18,7 +18,7 @@ public NativeLibraryMetadata? Metadata
         {
             get
             {
-                return new NativeLibraryMetadata(_libraryName, false, AvxLevel.None);
+                return new NativeLibraryMetadata(_libraryName, false, false, AvxLevel.None);
             }
         }
 

LLama/Native/Load/NativeLibraryWithVulkan.cs

@@ -0,0 +1,65 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+    /// <summary>
+    /// A native library compiled with vulkan.
+    /// </summary>
+    public class NativeLibraryWithVulkan : INativeLibrary
+    {
+        private string? _vulkanVersion;
+        private NativeLibraryName _libraryName;
+        private AvxLevel _avxLevel;
+        private bool _skipCheck;
+
+        /// <inheritdoc/>
+        public NativeLibraryMetadata? Metadata
+        {
+            get
+            {
+                return new NativeLibraryMetadata(_libraryName, false, true, _avxLevel);
+            }
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="vulkanVersion"></param>
+        /// <param name="libraryName"></param>
+        /// <param name="skipCheck"></param>
+        public NativeLibraryWithVulkan(string? vulkanVersion, NativeLibraryName libraryName, bool skipCheck)
+        {
+            _vulkanVersion = vulkanVersion;
+            _libraryName = libraryName;
+            _skipCheck = skipCheck;
+        }
+
+        /// <inheritdoc/>
+        public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            // TODO: Avx level is ignored now, needs to be implemented in the future.
+            if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck)
+            {
+                if(systemInfo.VulkanVersion != null)
+                {
+                    var vulkanLibraryPath = GetVulkanPath(systemInfo, logCallback);
+                    if (vulkanLibraryPath is not null)
+                    {
+                        yield return vulkanLibraryPath;
+                    }
+                }
+            }
+        }
+
+        private string? GetVulkanPath(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
+            var relativePath = $"runtimes/{os}/native/vulkan/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+            return relativePath;
+        }
+    }
+#endif
+}

LLama/Native/Load/SystemInfo.cs

@@ -1,7 +1,9 @@
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Runtime.InteropServices;
 using System.Text.Json;
+using System.Text.RegularExpressions;
 
 namespace LLama.Native
 {
@@ -10,7 +12,8 @@
     /// </summary>
     /// <param name="OSPlatform"></param>
     /// <param name="CudaMajorVersion"></param>
-    public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion)
+    /// <param name="VulkanVersion"></param>
+    public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion, string? VulkanVersion)
     {
         /// <summary>
         /// Get the system information of the current machine.
@@ -37,8 +40,108 @@
                 throw new PlatformNotSupportedException();
             }
 
-            return new SystemInfo(platform, GetCudaMajorVersion());
+            return new SystemInfo(platform, GetCudaMajorVersion(), GetVulkanVersion());
         }
+
+        #region Vulkan version
+        private static string? GetVulkanVersion()
+        {
+            // Get Vulkan Summary
+            string? vulkanSummary = GetVulkanSummary();
+
+            // If we have a Vulkan summary
+            if (vulkanSummary != null)
+            {
+                // Extract Vulkan version from summary
+                string? vulkanVersion = ExtractVulkanVersionFromSummary(vulkanSummary);
+
+                // If we have a Vulkan version
+                if (vulkanVersion != null)
+                {
+                    // Return the Vulkan version
+                    return vulkanVersion;
+                }
+            }
+
+            // Return null if we failed to get the Vulkan version
+            return null;
+        }
+
+        private static string? GetVulkanSummary()
+        {
+            // Note: on Linux, this requires `vulkan-tools` to be installed. (`sudo apt install vulkan-tools`)
+            try
+            {
+                // Set up the process start info
+                ProcessStartInfo start = new()
+                {
+                    FileName = "vulkaninfo",
+                    Arguments = "--summary",
+                    RedirectStandardOutput = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                };
+
+                // Start the process
+                Process process = new()
+                {
+                    StartInfo = start
+                };
+                process.Start();
+
+                // Read the output to a string
+                string output = process.StandardOutput.ReadToEnd();
+
+                // Wait for the process to exit
+                process.WaitForExit();
+
+                // Return the output
+                return output;
+            }
+            catch (Exception e)
+            {
+                //Console.WriteLine(e);
+
+                // Return null if we failed to get the Vulkan version
+                return null;
+            }
+        }
+
+        static string? ExtractVulkanVersionFromSummary(string vulkanSummary)
+        {
+            // We have three ways of parsing the Vulkan version from the summary (output is a different between Windows and Linux)
+            // For now, I have decided to go with the full version number, and leave it up to the user to parse it further if needed
+            // I have left the other patterns in, in case we need them in the future
+
+            // Output on linux : 4206847 (1.3.255)
+            // Output on windows : 1.3.255
+            string pattern = @"apiVersion\s*=\s*([^\r\n]+)";
+
+            // Output on linux : 4206847
+            // Output on windows : 1.3.255
+            //string pattern = @"apiVersion\s*=\s*([\d\.]+)";
+
+            // Output on linux : 1.3.255
+            // Output on windows : 1.3.255
+            //string pattern = @"apiVersion\s*=\s*(?:\d+\s*)?(?:\(\s*)?([\d]+\.[\d]+\.[\d]+)(?:\s*\))?";
+
+            // Create a Regex object to match the pattern
+            Regex regex = new Regex(pattern);
+
+            // Match the pattern in the input string
+            Match match = regex.Match(vulkanSummary);
+
+            // If a match is found
+            if (match.Success)
+            {
+                // Return the version number
+                return match.Groups[1].Value;
+            }
+
+            // Return null if no match is found
+            return null;
+        }
+        #endregion
 
         #region CUDA version
         private static int GetCudaMajorVersion()