Skip to content

Commit 597188c

Browse files
martindevansmartindevans
authored
Merge pull request #316 from martindevans/update_binaries_nov
November Binary Update
2 parents cf4d222 + 7486e63 commit 597188c

26 files changed

+329
-50
lines changed

‎LLama.Unittest/BasicTest.cs

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
using System.Text;
22
using LLama.Common;
3+
using LLama.Native;
4+
using Xunit.Abstractions;
35

46
namespace LLama.Unittest
57
{
68
public sealed class BasicTest
79
: IDisposable
810
{
11+
private readonly ITestOutputHelper _testOutputHelper;
912
private readonly ModelParams _params;
1013
private readonly LLamaWeights _model;
1114

12-
public BasicTest()
15+
public BasicTest(ITestOutputHelper testOutputHelper)
1316
{
17+
_testOutputHelper = testOutputHelper;
1418
_params = new ModelParams(Constants.ModelPath)
1519
{
1620
ContextSize = 2048
@@ -30,5 +34,57 @@ public void BasicModelProperties()
3034
Assert.Equal(4096, _model.ContextSize);
3135
Assert.Equal(4096, _model.EmbeddingSize);
3236
}
37+
38+
[Fact]
39+
public void AdvancedModelProperties()
40+
{
41+
var expected = new Dictionary<string, string>
42+
{
43+
{ "general.name", "LLaMA v2" },
44+
{ "general.architecture", "llama" },
45+
{ "general.quantization_version", "2" },
46+
{ "general.file_type", "2" },
47+
48+
{ "llama.context_length", "4096" },
49+
{ "llama.rope.dimension_count", "128" },
50+
{ "llama.embedding_length", "4096" },
51+
{ "llama.block_count", "32" },
52+
{ "llama.feed_forward_length", "11008" },
53+
{ "llama.attention.head_count", "32" },
54+
{ "llama.attention.head_count_kv", "32" },
55+
{ "llama.attention.layer_norm_rms_epsilon", "0.000001" },
56+
57+
{ "tokenizer.ggml.eos_token_id", "2" },
58+
{ "tokenizer.ggml.model", "llama" },
59+
{ "tokenizer.ggml.bos_token_id", "1" },
60+
{ "tokenizer.ggml.unknown_token_id", "0" },
61+
};
62+
63+
var metaCount = NativeApi.llama_model_meta_count(_model.NativeHandle);
64+
Assert.Equal(expected.Count, metaCount);
65+
66+
Span<byte> buffer = stackalloc byte[128];
67+
for (var i = 0; i < expected.Count; i++)
68+
{
69+
unsafe
70+
{
71+
fixed (byte* ptr = buffer)
72+
{
73+
var length = NativeApi.llama_model_meta_key_by_index(_model.NativeHandle, i, ptr, 128);
74+
Assert.True(length > 0);
75+
var key = Encoding.UTF8.GetString(buffer[..length]);
76+
77+
length = NativeApi.llama_model_meta_val_str_by_index(_model.NativeHandle, i, ptr, 128);
78+
Assert.True(length > 0);
79+
var val = Encoding.UTF8.GetString(buffer[..length]);
80+
81+
_testOutputHelper.WriteLine($"{key} == {val}");
82+
83+
Assert.True(expected.ContainsKey(key));
84+
Assert.Equal(expected[key], val);
85+
}
86+
}
87+
}
88+
}
3389
}
3490
}

‎LLama.Unittest/LLama.Unittest.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
<Nullable>enable</Nullable>
99

1010
<IsPackable>false</IsPackable>
11+
12+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
1113
</PropertyGroup>
1214

1315
<ItemGroup>

‎LLama/LLamaSharp.Runtime.targets

Lines changed: 64 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -3,41 +3,69 @@
33
<IncludeBuiltInRuntimes Condition="'$(IncludeBuiltInRuntimes)' == ''">true</IncludeBuiltInRuntimes>
44
</PropertyGroup>
55
<ItemGroup Condition="'$(IncludeBuiltInRuntimes)' == 'true'">
6-
<None Include="$(MSBuildThisFileDirectory)runtimes/libllama.dll">
7-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
8-
<Link>runtimes/win-x64/native/libllama.dll</Link>
9-
</None>
10-
<None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda11.dll">
11-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
12-
<Link>runtimes/win-x64/native/cuda11/libllama.dll</Link>
13-
</None>
14-
<None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda12.dll">
15-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
16-
<Link>runtimes/win-x64/native/cuda12/libllama.dll</Link>
17-
</None>
18-
<None Include="$(MSBuildThisFileDirectory)runtimes/libllama.so">
19-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
20-
<Link>runtimes/linux-x64/native/libllama.so</Link>
21-
</None>
22-
<None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda11.so">
23-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
24-
<Link>runtimes/linux-x64/native/cuda11/libllama.so</Link>
25-
</None>
26-
<None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda12.so">
27-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
28-
<Link>runtimes/linux-x64/native/cuda12/libllama.so</Link>
29-
</None>
30-
<None Include="$(MSBuildThisFileDirectory)runtimes/osx-arm64/libllama.dylib">
31-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
32-
<Link>runtimes/osx-arm64/native/libllama.dylib</Link>
33-
</None>
34-
<None Include="$(MSBuildThisFileDirectory)runtimes/osx-arm64/ggml-metal.metal">
35-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
36-
<Link>runtimes/osx-arm64/native/ggml-metal.metal</Link>
37-
</None>
38-
<None Include="$(MSBuildThisFileDirectory)runtimes/osx-x64/libllama.dylib">
39-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
40-
<Link>runtimes/osx-x64/native/libllama.dylib</Link>
41-
</None>
6+
7+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/libllama.dll">
8+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
9+
<Link>runtimes/win-x64/native/noavx/libllama.dll</Link>
10+
</None>
11+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/libllama.dll">
12+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
13+
<Link>runtimes/win-x64/native/avx/libllama.dll</Link>
14+
</None>
15+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/libllama.dll">
16+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
17+
<Link>runtimes/win-x64/native/avx2/libllama.dll</Link>
18+
</None>
19+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/libllama.dll">
20+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
21+
<Link>runtimes/win-x64/native/avx512/libllama.dll</Link>
22+
</None>
23+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libllama.dll">
24+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
25+
<Link>runtimes/win-x64/native/cuda11/libllama.dll</Link>
26+
</None>
27+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.1.0/libllama.dll">
28+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
29+
<Link>runtimes/win-x64/native/cuda12/libllama.dll</Link>
30+
</None>
31+
32+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/libllama.so">
33+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
34+
<Link>runtimes/linux-x64/native/noavx/libllama.so</Link>
35+
</None>
36+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/libllama.so">
37+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
38+
<Link>runtimes/linux-x64/native/avx/libllama.so</Link>
39+
</None>
40+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/libllama.so">
41+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
42+
<Link>runtimes/linux-x64/native/avx2/libllama.so</Link>
43+
</None>
44+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/libllama.so">
45+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
46+
<Link>runtimes/linux-x64/native/avx512/libllama.so</Link>
47+
</None>
48+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libllama.so">
49+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
50+
<Link>runtimes/linux-x64/native/cuda11/libllama.so</Link>
51+
</None>
52+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.1.0/libllama.so">
53+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
54+
<Link>runtimes/linux-x64/native/cuda12/libllama.so</Link>
55+
</None>
56+
57+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libllama.dylib">
58+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
59+
<Link>runtimes/osx-arm64/native/libllama.dylib</Link>
60+
</None>
61+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/ggml-metal.metal">
62+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
63+
<Link>runtimes/osx-arm64/native/ggml-metal.metal</Link>
64+
</None>
65+
66+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64/libllama.dylib">
67+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
68+
<Link>runtimes/osx-x64/native/libllama.dylib</Link>
69+
</None>
4270
</ItemGroup>
4371
</Project>

‎LLama/Native/NativeApi.cs

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,20 @@ public static int llama_tokenize(SafeLLamaContextHandle ctx, string text, Encodi
302302
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
303303
public static extern llama_token llama_token_nl(SafeLlamaModelHandle model);
304304

305+
/// <summary>
306+
/// Returns -1 if unknown, 1 for true or 0 for false.
307+
/// </summary>
308+
/// <returns></returns>
309+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
310+
public static extern int llama_add_bos_token(SafeLlamaModelHandle model);
311+
312+
/// <summary>
313+
/// Returns -1 if unknown, 1 for true or 0 for false.
314+
/// </summary>
315+
/// <returns></returns>
316+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
317+
public static extern int llama_add_eos_token(SafeLlamaModelHandle model);
318+
305319
/// <summary>
306320
/// Print out timing information for this context
307321
/// </summary>
@@ -348,18 +362,77 @@ public static int llama_tokenize(SafeLLamaContextHandle ctx, string text, Encodi
348362
public static extern int llama_n_embd(SafeLlamaModelHandle model);
349363

350364
/// <summary>
351-
/// Get the size of the model in bytes
365+
/// Get the model's RoPE frequency scaling factor
366+
/// </summary>
367+
/// <param name="model"></param>
368+
/// <returns></returns>
369+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
370+
public static extern float llama_rope_freq_scale_train(SafeLlamaModelHandle model);
371+
372+
/// <summary>
373+
/// Get metadata value as a string by key name
374+
/// </summary>
375+
/// <param name="model"></param>
376+
/// <param name="key"></param>
377+
/// <param name="buf"></param>
378+
/// <param name="buf_size"></param>
379+
/// <returns>The length of the string on success, or -1 on failure</returns>
380+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
381+
public static extern int llama_model_meta_val_str(SafeLlamaModelHandle model, byte* key, byte* buf, long buf_size);
382+
383+
/// <summary>
384+
/// Get the number of metadata key/value pairs
352385
/// </summary>
353386
/// <param name="model"></param>
354387
/// <returns></returns>
355388
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
389+
public static extern int llama_model_meta_count(SafeLlamaModelHandle model);
390+
391+
/// <summary>
392+
/// Get metadata key name by index
393+
/// </summary>
394+
/// <param name="model"></param>
395+
/// <param name="index"></param>
396+
/// <param name="buf"></param>
397+
/// <param name="buf_size"></param>
398+
/// <returns>The length of the string on success, or -1 on failure</returns>
399+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
400+
public static extern int llama_model_meta_key_by_index(SafeLlamaModelHandle model, int index, byte* buf, long buf_size);
401+
402+
/// <summary>
403+
/// Get metadata value as a string by index
404+
/// </summary>
405+
/// <param name="model"></param>
406+
/// <param name="index"></param>
407+
/// <param name="buf"></param>
408+
/// <param name="buf_size"></param>
409+
/// <returns>The length of the string on success, or -1 on failure</returns>
410+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
411+
public static extern int llama_model_meta_val_str_by_index(SafeLlamaModelHandle model, int index, byte* buf, long buf_size);
412+
413+
/// <summary>
414+
/// Get a string describing the model type
415+
/// </summary>
416+
/// <param name="model"></param>
417+
/// <param name="buf"></param>
418+
/// <param name="buf_size"></param>
419+
/// <returns>The length of the string on success, or -1 on failure</returns>
420+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
421+
public static extern int llama_model_desc(SafeLlamaModelHandle model, byte* buf, long buf_size);
422+
423+
/// <summary>
424+
/// Get the size of the model in bytes
425+
/// </summary>
426+
/// <param name="model"></param>
427+
/// <returns>The size of the model</returns>
428+
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
356429
public static extern ulong llama_model_size(SafeLlamaModelHandle model);
357430

358431
/// <summary>
359432
/// Get the number of parameters in this model
360433
/// </summary>
361434
/// <param name="model"></param>
362-
/// <returns></returns>
435+
/// <returns>The functions return the length of the string on success, or -1 on failure</returns>
363436
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
364437
public static extern ulong llama_model_n_params(SafeLlamaModelHandle model);
365438

@@ -370,7 +443,7 @@ public static int llama_tokenize(SafeLLamaContextHandle ctx, string text, Encodi
370443
/// <param name="llamaToken"></param>
371444
/// <param name="buffer">buffer to write string into</param>
372445
/// <param name="length">size of the buffer</param>
373-
/// <returns>The length writte, or if the buffer is too small a negative that indicates the length required</returns>
446+
/// <returns>The length written, or if the buffer is too small a negative that indicates the length required</returns>
374447
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
375448
public static extern int llama_token_to_piece(SafeLlamaModelHandle model, int llamaToken, byte* buffer, int length);
376449

‎LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,21 @@
1717

1818
<files>
1919
<file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cpu.props" />
20-
<file src="runtimes/libllama.dll" target="runtimes\win-x64\native\libllama.dll" />
21-
<file src="runtimes/libllama.so" target="runtimes\linux-x64\native\libllama.so" />
22-
<file src="runtimes/osx-x64/libllama.dylib" target="runtimes\osx-x64\native\libllama.dylib" />
23-
<file src="runtimes/osx-arm64/libllama.dylib" target="runtimes\osx-arm64\native\libllama.dylib" />
24-
<file src="runtimes/osx-arm64/ggml-metal.metal" target="runtimes\osx-arm64\native\ggml-metal.metal" />
20+
21+
<file src="runtimes/deps/libllama.dll" target="runtimes\win-x64\native\libllama.dll" />
22+
<file src="runtimes/deps/avx/libllama.dll" target="runtimes\win-x64\native\avx\libllama.dll" />
23+
<file src="runtimes/deps/avx2/libllama.dll" target="runtimes\win-x64\native\avx2\libllama.dll" />
24+
<file src="runtimes/deps/avx512/libllama.dll" target="runtimes\win-x64\native\avx512\libllama.dll" />
25+
26+
<file src="runtimes/deps/libllama.so" target="runtimes\linux-x64\native\libllama.so" />
27+
<file src="runtimes/deps/avx/libllama.so" target="runtimes\linux-x64\native\avx\libllama.so" />
28+
<file src="runtimes/deps/avx2/libllama.so" target="runtimes\linux-x64\native\avx2\libllama.so" />
29+
<file src="runtimes/deps/avx512/libllama.so" target="runtimes\linux-x64\native\avx512\libllama.so" />
30+
31+
<file src="runtimes/deps/osx-x64/libllama.dylib" target="runtimes\osx-x64\native\libllama.dylib" />
32+
<file src="runtimes/deps/osx-arm64/libllama.dylib" target="runtimes\osx-arm64\native\libllama.dylib" />
33+
<file src="runtimes/deps/osx-arm64/ggml-metal.metal" target="runtimes\osx-arm64\native\ggml-metal.metal" />
34+
2535
<file src="icon512.png" target="icon512.png" />
2636
</files>
2737
</package>

‎LLama/runtimes/build/LLamaSharp.Backend.Cuda11.nuspec

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717

1818
<files>
1919
<file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda11.props" />
20-
<file src="runtimes/libllama-cuda11.dll" target="runtimes\win-x64\native\cuda11\libllama.dll" />
21-
<file src="runtimes/libllama-cuda11.so" target="runtimes\linux-x64\native\cuda11\libllama.so" />
20+
21+
<file src="runtimes/deps/cu11.7.1/libllama.dll" target="runtimes\win-x64\native\cuda11\libllama.dll" />
22+
<file src="runtimes/deps/cu11.7.1/libllama.so" target="runtimes\linux-x64\native\cuda11\libllama.so" />
23+
2224
<file src="icon512.png" target="icon512.png" />
2325
</files>
2426
</package>

‎LLama/runtimes/build/LLamaSharp.Backend.Cuda12.nuspec

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717

1818
<files>
1919
<file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda12.props" />
20-
<file src="runtimes/libllama-Cuda12.dll" target="runtimes\win-x64\native\cuda12\libllama.dll" />
21-
<file src="runtimes/libllama-Cuda12.so" target="runtimes\linux-x64\native\cuda12\libllama.so" />
20+
21+
<file src="runtimes/deps/cu12.1.0/libllama.dll" target="runtimes\win-x64\native\cuda12\libllama.dll" />
22+
<file src="runtimes/deps/cu12.1.0/libllama.so" target="runtimes\linux-x64\native\cuda12\libllama.so" />
23+
2224
<file src="icon512.png" target="icon512.png" />
2325
</files>
2426
</package>

‎LLama/runtimes/deps/avx/libllama.dll

937 KB
Binary file not shown.

‎LLama/runtimes/deps/avx/libllama.so

964 KB
Binary file not shown.

‎LLama/runtimes/deps/avx2/libllama.dll

921 KB
Binary file not shown.

‎LLama/runtimes/deps/avx2/libllama.so

964 KB
Binary file not shown.

‎LLama/runtimes/deps/avx512/libllama.dll

921 KB
Binary file not shown.

‎LLama/runtimes/deps/avx512/libllama.so

1000 KB
Binary file not shown.

‎LLama/runtimes/libllama-cuda11.dll renamed to ‎LLama/runtimes/deps/cu11.7.1/libllama.dll

3.21 MB
Binary file not shown.

‎LLama/runtimes/libllama-cuda11.so renamed to ‎LLama/runtimes/deps/cu11.7.1/libllama.so

3.32 MB
Binary file not shown.

‎LLama/runtimes/libllama-cuda12.dll renamed to ‎LLama/runtimes/deps/cu12.1.0/libllama.dll

3.18 MB
Binary file not shown.

‎LLama/runtimes/libllama-cuda12.so renamed to ‎LLama/runtimes/deps/cu12.1.0/libllama.so

3.28 MB
Binary file not shown.

‎LLama/runtimes/deps/libllama.dll

932 KB
Binary file not shown.

‎LLama/runtimes/deps/libllama.so

964 KB
Binary file not shown.

0 commit comments

Comments
 (0)