(#271) Semantic/Vector Search

Author: phongnguyend

src/Monolith/ClassifiedAds.Blazor.Modules/Core/Pages/Index.razor

@@ -32,5 +32,5 @@
 </div>
 
 @code{
-    string version = "7.0";
+    string version = "10.0";
 }

src/Monolith/ClassifiedAds.Blazor.Modules/Files/Pages/Edit.cs

@@ -28,6 +28,8 @@ public partial class Edit
 
     protected AuditLogsDialog AuditLogsDialog { get; set; }
 
+    public bool ShowTokenDetails { get; set; }
+
     protected override async Task OnInitializedAsync()
     {
         File = await FileService.GetFileByIdAsync(Id);
@@ -45,10 +47,22 @@ protected async Task HandleValidSubmit()
         NavManager.NavigateTo("/files");
     }
 
+    protected async Task DownloadText()
+    {
+        var ext = Path.GetExtension(File.FileEntryText.TextLocation).ToLowerInvariant();
+        var token = await FileService.GetAccessToken();
+        await JSRuntime.InvokeVoidAsync("interop.downloadFile", FileService.GetDownloadTextUrl(File.Id), token, File.FileName + ext);
+    }
+
+    protected async Task DownloadChunk(FileEntryEmbeddingModel chunk)
+    {
+        var token = await FileService.GetAccessToken();
+        await JSRuntime.InvokeVoidAsync("interop.downloadFile", FileService.GetDownloadChunkUrl(File.Id, chunk.ChunkName), token, chunk.ChunkName);
+    }
+
     protected async Task DownloadEmbedding(FileEntryEmbeddingModel embedding)
     {
         using var streamRef = new DotNetStreamReference(stream: new MemoryStream(Encoding.UTF8.GetBytes(embedding.Embedding)));
-
         await JSRuntime.InvokeVoidAsync("interop.downloadFileFromStream", "Embedding.txt", streamRef);
     }
 }

src/Monolith/ClassifiedAds.Blazor.Modules/Files/Pages/Edit.razor

@@ -47,7 +47,7 @@
 				<div class="form-group row">
 					<label for="textLocation" class="col-sm-2 col-form-label">Text Location</label>
 					<div class="col-sm-10">
-						@File.FileEntryText?.TextLocation
+						<a href="javascript:none()" @onclick="(() => DownloadText())">@File.FileEntryText?.TextLocation</a>
 					</div>
 				</div>
 			}
@@ -80,25 +80,36 @@
 		</div>
 
 		<div class="card-body">
+			<div style="float: right;">
+				<InputCheckbox id="showTokenDetails" @bind-Value="ShowTokenDetails" /> Show Token Details
+			</div>
 			<table class="table">
 				<thead>
 					<tr>
 						<th>Chunk Name</th>
 						<th>Chunk Location</th>
-						<th>Embedding</th>
-						<th>Token Details</th>
 						<th>Created Date Time</th>
+						<th>Embedding</th>
+						@if (ShowTokenDetails)
+						{
+							<th>Token Details</th>
+						}
 					</tr>
 				</thead>
 				<tbody>
 					@foreach (var fileEntryEmbedding in File.FileEntryEmbeddings)
 					{
 						<tr>
 							<td>@fileEntryEmbedding.ChunkName</td>
-							<td>@fileEntryEmbedding.ChunkLocation</td>
-							<td><a href="javascript:none()" @onclick="(() => DownloadEmbedding(fileEntryEmbedding))">Download</a></td>
-							<td>@fileEntryEmbedding.TokenDetails</td>
+							<td>
+								<a href="javascript:none()" @onclick="(() => DownloadChunk(fileEntryEmbedding))">@fileEntryEmbedding.ChunkLocation</a>
+							</td>
 							<td>@fileEntryEmbedding.CreatedDateTime</td>
+							<td><a href="javascript:none()" @onclick="(() => DownloadEmbedding(fileEntryEmbedding))">Download</a></td>
+							@if (ShowTokenDetails)
+							{
+								<td>@fileEntryEmbedding.TokenDetails</td>
+							}
 						</tr>
 					}
 				</tbody>

src/Monolith/ClassifiedAds.Blazor.Modules/Files/Services/FileService.cs

@@ -21,6 +21,16 @@ public string GetDownloadUrl(Guid id)
         return $"{_httpClient.BaseAddress.AbsoluteUri.Trim('/')}/api/files/{id}/download";
     }
 
+    public string GetDownloadTextUrl(Guid id)
+    {
+        return $"{_httpClient.BaseAddress.AbsoluteUri.Trim('/')}/api/files/{id}/downloadtext";
+    }
+
+    public string GetDownloadChunkUrl(Guid id, string chunkName)
+    {
+        return $"{_httpClient.BaseAddress.AbsoluteUri.Trim('/')}/api/files/{id}/downloadchunk/{chunkName}";
+    }
+
     public string GetUploadUrl()
     {
         return $"{_httpClient.BaseAddress.AbsoluteUri.Trim('/')}/api/files";

src/Monolith/ClassifiedAds.WebAPI/Controllers/FilesController.cs

@@ -162,6 +162,7 @@ public async Task<ActionResult<IEnumerable<FileEntryModel>>> Get(Guid id)
 
         model.FileEntryEmbeddings = _fileEntryEmbeddingRepository.GetQueryableSet()
             .Where(x => x.FileEntryId == fileEntry.Id)
+            .OrderBy(x => x.CreatedDateTime)
             .Select(x => new FileEntryEmbeddingModel
             {
                 ChunkName = x.ChunkName,
@@ -170,7 +171,8 @@ public async Task<ActionResult<IEnumerable<FileEntryModel>>> Get(Guid id)
                 TokenDetails = x.TokenDetails,
                 CreatedDateTime = x.CreatedDateTime,
                 UpdatedDateTime = x.UpdatedDateTime,
-            }).ToList();
+            })
+            .ToList();
 
         return Ok(model);
     }
@@ -213,6 +215,53 @@ public async Task<IActionResult> Download(Guid id)
         return File(content, MediaTypeNames.Application.Octet, WebUtility.HtmlEncode(fileEntry.FileName));
     }
 
+    [Authorize(Permissions.DownloadFile)]
+    [HttpGet("{id}/downloadtext")]
+    public async Task<IActionResult> DownloadText(Guid id)
+    {
+        var fileEntry = await _dispatcher.DispatchAsync(new GetEntityByIdQuery<FileEntry> { Id = id });
+
+        var authorizationResult = await _authorizationService.AuthorizeAsync(User, fileEntry, Operations.Read);
+        if (!authorizationResult.Succeeded)
+        {
+            return Forbid();
+        }
+
+        var fileEntryText = _fileEntryTextRepository.GetQueryableSet()
+              .Where(x => x.FileEntryId == fileEntry.Id)
+              .Select(x => new FileEntryTextModel
+              {
+                  TextLocation = x.TextLocation
+              }).FirstOrDefault();
+
+        var stream = System.IO.File.OpenRead(Path.Combine(_options.Storage.TempFolderPath, fileEntryText.TextLocation));
+        var ext = Path.GetExtension(fileEntryText.TextLocation).ToLowerInvariant();
+        return File(stream, MediaTypeNames.Application.Octet, WebUtility.HtmlEncode(fileEntry.FileName + ext));
+    }
+
+    [Authorize(Permissions.DownloadFile)]
+    [HttpGet("{id}/downloadchunk/{chunkName}")]
+    public async Task<IActionResult> DownloadChunk(Guid id, string chunkName)
+    {
+        var fileEntry = await _dispatcher.DispatchAsync(new GetEntityByIdQuery<FileEntry> { Id = id });
+
+        var authorizationResult = await _authorizationService.AuthorizeAsync(User, fileEntry, Operations.Read);
+        if (!authorizationResult.Succeeded)
+        {
+            return Forbid();
+        }
+
+        var fileEntryEmbedding = _fileEntryEmbeddingRepository.GetQueryableSet()
+              .Where(x => x.FileEntryId == fileEntry.Id && x.ChunkName == chunkName)
+              .Select(x => new FileEntryEmbeddingModel
+              {
+                  ChunkLocation = x.ChunkLocation,
+              }).FirstOrDefault();
+
+        var stream = System.IO.File.OpenRead(Path.Combine(_options.Storage.TempFolderPath, fileEntryEmbedding.ChunkLocation));
+        return File(stream, MediaTypeNames.Application.Octet, WebUtility.HtmlEncode(fileEntryEmbedding.ChunkName));
+    }
+
     [Authorize(Permissions.UpdateFile)]
     [HttpPut("{id}")]
     [Consumes("application/json")]

src/Monolith/ClassifiedAds.WebAPI/appsettings.json

@@ -99,6 +99,7 @@
   "Storage": {
     "Provider": "Local",
     "MasterEncryptionKey": "+2ZC9wrwlvPswPxCND0BjrKJ3CfOpImGtn4hloVwo2I=",
+    "TempFolderPath": "C:\\Data\\Practical.CleanArchitecture\\Temp",
     "Local": {
       "Path": "C:\\Data\\files"
     },