Skip to content

Commit e907146

Browse files
m0nskym0nsky
authored
Vulkan support (#797)
* Initial Vulkan set up * Replaced old cublas/cuda mention with vulkan * Make sure we use the correct llama_cpp_commit for Vulkan * Remove duplicate cmake params (already defined in COMMON_DEFINE) * Add llava & ggml to Vulkan nuspec * Updated runtime targets for Vulkan (llava + ggml) * Make Vulkan nuspec match the CUDA nuspec title/order
1 parent 6598121 commit e907146

10 files changed

+270
-9
lines changed

‎LLama/LLamaSharp.Runtime.targets

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,14 @@
5252
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
5353
<Link>runtimes/win-x64/native/cuda12/ggml.dll</Link>
5454
</None>
55+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/llama.dll">
56+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
57+
<Link>runtimes/win-x64/native/vulkan/llama.dll</Link>
58+
</None>
59+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/ggml.dll">
60+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
61+
<Link>runtimes/win-x64/native/vulkan/ggml.dll</Link>
62+
</None>
5563

5664
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/libllama.so">
5765
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -101,6 +109,14 @@
101109
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
102110
<Link>runtimes/linux-x64/native/cuda12/libggml.so</Link>
103111
</None>
112+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libllama.so">
113+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
114+
<Link>runtimes/linux-x64/native/vulkan/libllama.so</Link>
115+
</None>
116+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libggml.so">
117+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
118+
<Link>runtimes/linux-x64/native/vulkan/libggml.so</Link>
119+
</None>
104120

105121
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libggml.dylib">
106122
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -169,6 +185,10 @@
169185
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
170186
<Link>runtimes/win-x64/native/cuda12/llava_shared.dll</Link>
171187
</None>
188+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/llava_shared.dll">
189+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
190+
<Link>runtimes/win-x64/native/vulkan/llava_shared.dll</Link>
191+
</None>
172192

173193
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/libllava_shared.so">
174194
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -194,5 +214,9 @@
194214
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
195215
<Link>runtimes/linux-x64/native/cuda12/libllava_shared.so</Link>
196216
</None>
217+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libllava_shared.so">
218+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
219+
<Link>runtimes/linux-x64/native/vulkan/libllava_shared.so</Link>
220+
</None>
197221
</ItemGroup>
198222
</Project>

‎LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,12 @@ public IEnumerable<INativeLibrary> Apply(NativeLibraryConfig.Description descrip
2828
yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck);
2929
}
3030

31-
if(!description.UseCuda || description.AllowFallback)
31+
if (description.UseVulkan)
32+
{
33+
yield return new NativeLibraryWithVulkan(systemInfo.VulkanVersion, description.Library, description.SkipCheck);
34+
}
35+
36+
if((!description.UseCuda || !description.UseVulkan) || description.AllowFallback)
3237
{
3338
if (description.AllowFallback)
3439
{

‎LLama/Native/Load/NativeLibraryConfig.cs

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ public sealed partial class NativeLibraryConfig
1616
private string? _libraryPath;
1717

1818
private bool _useCuda = true;
19+
private bool _useVulkan = true;
1920
private AvxLevel _avxLevel;
2021
private bool _allowFallback = true;
2122
private bool _skipCheck = false;
@@ -55,6 +56,20 @@ public NativeLibraryConfig WithCuda(bool enable = true)
5556
_useCuda = enable;
5657
return this;
5758
}
59+
60+
/// <summary>
61+
/// Configure whether to use vulkan backend if possible. Default is true.
62+
/// </summary>
63+
/// <param name="enable"></param>
64+
/// <returns></returns>
65+
/// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
66+
public NativeLibraryConfig WithVulkan(bool enable = true)
67+
{
68+
ThrowIfLoaded();
69+
70+
_useVulkan = enable;
71+
return this;
72+
}
5873

5974
/// <summary>
6075
/// Configure the prefferred avx support level of the backend.
@@ -159,6 +174,7 @@ internal Description CheckAndGatherDescription()
159174
path,
160175
NativeLibraryName,
161176
_useCuda,
177+
_useVulkan,
162178
_avxLevel,
163179
_allowFallback,
164180
_skipCheck,
@@ -229,7 +245,7 @@ private static bool CheckAVX512()
229245
/// <param name="AllowFallback"></param>
230246
/// <param name="SkipCheck"></param>
231247
/// <param name="SearchDirectories"></param>
232-
public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck,
248+
public record Description(string? Path, NativeLibraryName Library, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck,
233249
string[] SearchDirectories)
234250
{
235251
/// <inheritdoc/>
@@ -250,6 +266,7 @@ public override string ToString()
250266
$"- LibraryName: {Library}\n" +
251267
$"- Path: '{Path}'\n" +
252268
$"- PreferCuda: {UseCuda}\n" +
269+
$"- PreferVulkan: {UseVulkan}\n" +
253270
$"- PreferredAvxLevel: {avxLevelString}\n" +
254271
$"- AllowFallback: {AllowFallback}\n" +
255272
$"- SkipCheck: {SkipCheck}\n" +
@@ -425,6 +442,21 @@ public NativeLibraryConfigContainer WithCuda(bool enable = true)
425442
}
426443
return this;
427444
}
445+
446+
/// <summary>
447+
/// Configure whether to use vulkan backend if possible.
448+
/// </summary>
449+
/// <param name="enable"></param>
450+
/// <returns></returns>
451+
/// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
452+
public NativeLibraryConfigContainer WithVulkan(bool enable = true)
453+
{
454+
foreach(var config in _configs)
455+
{
456+
config.WithVulkan(enable);
457+
}
458+
return this;
459+
}
428460

429461
/// <summary>
430462
/// Configure the prefferred avx support level of the backend.

‎LLama/Native/Load/NativeLibraryMetadata.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@ namespace LLama.Native
66
/// </summary>
77
/// <param name="NativeLibraryName">Which kind of library it is.</param>
88
/// <param name="UseCuda">Whether it's compiled with cublas.</param>
9+
/// <param name="UseVulkan">Whether it's compiled with vulkan.</param>
910
/// <param name="AvxLevel">Which AvxLevel it's compiled with.</param>
10-
public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, AvxLevel AvxLevel)
11+
public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel)
1112
{
1213
public override string ToString()
1314
{
14-
return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, AvxLevel: {AvxLevel})";
15+
return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, UseVulkan: {UseVulkan}, AvxLevel: {AvxLevel})";
1516
}
1617
}
1718

‎LLama/Native/Load/NativeLibraryWithAvx.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public NativeLibraryMetadata? Metadata
1919
{
2020
get
2121
{
22-
return new NativeLibraryMetadata(_libraryName, false, _avxLevel);
22+
return new NativeLibraryMetadata(_libraryName, false, false,_avxLevel);
2323
}
2424
}
2525

‎LLama/Native/Load/NativeLibraryWithCuda.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public NativeLibraryMetadata? Metadata
2020
{
2121
get
2222
{
23-
return new NativeLibraryMetadata(_libraryName, true, _avxLevel);
23+
return new NativeLibraryMetadata(_libraryName, true, false, _avxLevel);
2424
}
2525
}
2626

‎LLama/Native/Load/NativeLibraryWithMacOrFallback.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public NativeLibraryMetadata? Metadata
1818
{
1919
get
2020
{
21-
return new NativeLibraryMetadata(_libraryName, false, AvxLevel.None);
21+
return new NativeLibraryMetadata(_libraryName, false, false, AvxLevel.None);
2222
}
2323
}
2424

‎LLama/Native/Load/NativeLibraryWithVulkan.cs

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using LLama.Abstractions;
2+
using System.Collections.Generic;
3+
using System.Runtime.InteropServices;
4+
5+
namespace LLama.Native
6+
{
7+
#if NET6_0_OR_GREATER
8+
/// <summary>
9+
/// A native library compiled with vulkan.
10+
/// </summary>
11+
public class NativeLibraryWithVulkan : INativeLibrary
12+
{
13+
private string? _vulkanVersion;
14+
private NativeLibraryName _libraryName;
15+
private AvxLevel _avxLevel;
16+
private bool _skipCheck;
17+
18+
/// <inheritdoc/>
19+
public NativeLibraryMetadata? Metadata
20+
{
21+
get
22+
{
23+
return new NativeLibraryMetadata(_libraryName, false, true, _avxLevel);
24+
}
25+
}
26+
27+
/// <summary>
28+
///
29+
/// </summary>
30+
/// <param name="vulkanVersion"></param>
31+
/// <param name="libraryName"></param>
32+
/// <param name="skipCheck"></param>
33+
public NativeLibraryWithVulkan(string? vulkanVersion, NativeLibraryName libraryName, bool skipCheck)
34+
{
35+
_vulkanVersion = vulkanVersion;
36+
_libraryName = libraryName;
37+
_skipCheck = skipCheck;
38+
}
39+
40+
/// <inheritdoc/>
41+
public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
42+
{
43+
// TODO: Avx level is ignored now, needs to be implemented in the future.
44+
if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck)
45+
{
46+
if(systemInfo.VulkanVersion != null)
47+
{
48+
var vulkanLibraryPath = GetVulkanPath(systemInfo, logCallback);
49+
if (vulkanLibraryPath is not null)
50+
{
51+
yield return vulkanLibraryPath;
52+
}
53+
}
54+
}
55+
}
56+
57+
private string? GetVulkanPath(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
58+
{
59+
NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
60+
var relativePath = $"runtimes/{os}/native/vulkan/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
61+
return relativePath;
62+
}
63+
}
64+
#endif
65+
}

‎LLama/Native/Load/SystemInfo.cs

Lines changed: 105 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
using System;
2+
using System.Diagnostics;
23
using System.IO;
34
using System.Runtime.InteropServices;
45
using System.Text.Json;
6+
using System.Text.RegularExpressions;
57

68
namespace LLama.Native
79
{
@@ -10,7 +12,8 @@ namespace LLama.Native
1012
/// </summary>
1113
/// <param name="OSPlatform"></param>
1214
/// <param name="CudaMajorVersion"></param>
13-
public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion)
15+
/// <param name="VulkanVersion"></param>
16+
public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion, string? VulkanVersion)
1417
{
1518
/// <summary>
1619
/// Get the system information of the current machine.
@@ -37,8 +40,108 @@ public static SystemInfo Get()
3740
throw new PlatformNotSupportedException();
3841
}
3942

40-
return new SystemInfo(platform, GetCudaMajorVersion());
43+
return new SystemInfo(platform, GetCudaMajorVersion(), GetVulkanVersion());
4144
}
45+
46+
#region Vulkan version
47+
private static string? GetVulkanVersion()
48+
{
49+
// Get Vulkan Summary
50+
string? vulkanSummary = GetVulkanSummary();
51+
52+
// If we have a Vulkan summary
53+
if (vulkanSummary != null)
54+
{
55+
// Extract Vulkan version from summary
56+
string? vulkanVersion = ExtractVulkanVersionFromSummary(vulkanSummary);
57+
58+
// If we have a Vulkan version
59+
if (vulkanVersion != null)
60+
{
61+
// Return the Vulkan version
62+
return vulkanVersion;
63+
}
64+
}
65+
66+
// Return null if we failed to get the Vulkan version
67+
return null;
68+
}
69+
70+
private static string? GetVulkanSummary()
71+
{
72+
// Note: on Linux, this requires `vulkan-tools` to be installed. (`sudo apt install vulkan-tools`)
73+
try
74+
{
75+
// Set up the process start info
76+
ProcessStartInfo start = new()
77+
{
78+
FileName = "vulkaninfo",
79+
Arguments = "--summary",
80+
RedirectStandardOutput = true,
81+
UseShellExecute = false,
82+
CreateNoWindow = true
83+
};
84+
85+
// Start the process
86+
Process process = new()
87+
{
88+
StartInfo = start
89+
};
90+
process.Start();
91+
92+
// Read the output to a string
93+
string output = process.StandardOutput.ReadToEnd();
94+
95+
// Wait for the process to exit
96+
process.WaitForExit();
97+
98+
// Return the output
99+
return output;
100+
}
101+
catch (Exception e)
102+
{
103+
//Console.WriteLine(e);
104+
105+
// Return null if we failed to get the Vulkan version
106+
return null;
107+
}
108+
}
109+
110+
static string? ExtractVulkanVersionFromSummary(string vulkanSummary)
111+
{
112+
// We have three ways of parsing the Vulkan version from the summary (output is a different between Windows and Linux)
113+
// For now, I have decided to go with the full version number, and leave it up to the user to parse it further if needed
114+
// I have left the other patterns in, in case we need them in the future
115+
116+
// Output on linux : 4206847 (1.3.255)
117+
// Output on windows : 1.3.255
118+
string pattern = @"apiVersion\s*=\s*([^\r\n]+)";
119+
120+
// Output on linux : 4206847
121+
// Output on windows : 1.3.255
122+
//string pattern = @"apiVersion\s*=\s*([\d\.]+)";
123+
124+
// Output on linux : 1.3.255
125+
// Output on windows : 1.3.255
126+
//string pattern = @"apiVersion\s*=\s*(?:\d+\s*)?(?:\(\s*)?([\d]+\.[\d]+\.[\d]+)(?:\s*\))?";
127+
128+
// Create a Regex object to match the pattern
129+
Regex regex = new Regex(pattern);
130+
131+
// Match the pattern in the input string
132+
Match match = regex.Match(vulkanSummary);
133+
134+
// If a match is found
135+
if (match.Success)
136+
{
137+
// Return the version number
138+
return match.Groups[1].Value;
139+
}
140+
141+
// Return null if no match is found
142+
return null;
143+
}
144+
#endregion
42145

43146
#region CUDA version
44147
private static int GetCudaMajorVersion()

0 commit comments

Comments
 (0)