Skip to content

Commit 829782d

Browse files
ljcollins25ljcollins25
committed
Merge remote-tracking branch 'origin/opensrc/main' into open-main
2 parents 98865e6 + 5c962e0 commit 829782d

File tree

8 files changed

+127
-77
lines changed

8 files changed

+127
-77
lines changed

‎src/Codex.Application/Verbs/UploadOperation.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,10 @@ protected override async ValueTask<int> ExecuteAsync()
4747
{
4848
if (!string.IsNullOrEmpty(BlobContainerSasUrl))
4949
{
50-
var storage = new BlobObjectStorage(Logger, BlobContainerSasUrl, Root: "");
50+
var storage = new BlobObjectStorage(Logger, BlobContainerSasUrl);
5151
storage.Initialize();
5252

53-
Logger.LogMessage($"Uploading to '{storage.Client.AccountName}' blob container '{storage.Client.Name}'");
53+
Logger.LogMessage($"Uploading to '{storage.Folder.AccountName}' blob folder '{storage.Folder.BlobContainerName}'");
5454

5555
await storage.UploadDirectory(UploadDirectory, ExcludedFiles);
5656
}

‎src/Codex.Integration.Tests/AnalyzeTestProject.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
namespace Codex.Integration.Tests;
3636

3737
using static CodexConstants;
38-
public record AnalyzeTestProject(ITestOutputHelper Output) : AnalyzeTestProjectBase(Output)
38+
public partial record AnalyzeTestProject(ITestOutputHelper Output) : AnalyzeTestProjectBase(Output)
3939
{
4040
[Fact]
4141
public async Task TestReindexClean()

‎src/Codex.Integration.Tests/AnalyzeTestProjectBase.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public record AnalyzeTestProjectOptions
3939
// Disabling till we fix zip upload / reading to yield results consistent with
4040
// file upload. Seems like content is changed somehow.
4141
public bool DisableBlockZipUpload = true;
42+
public bool UploadFiles = false;
4243
public int ProjectIndex = 0;
4344
public ITestProjectData Project => Projects[ProjectIndex];
4445
public string ProjectDirectory => Project.ProjectDirectory;

‎src/Codex.Integration.Tests/CodexTestBase.cs‎

Lines changed: 88 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Diagnostics;
22
using System.Runtime.CompilerServices;
3+
using Azure.Storage.Blobs;
34
using Codex.Application.Verbs;
45
using Codex.Lucene.Search;
56
using Codex.Sdk;
@@ -143,7 +144,7 @@ public async Task<WebProgramBase> CreateWebProgram(IngestOperation indexOperatio
143144

144145
await program.RunAsync();
145146

146-
return program; ;
147+
return program;
147148
}
148149

149150
public CodexPage CreateCodexApp(IngestOperation indexOperation,
@@ -217,86 +218,113 @@ public IReadOnlyStableIdStorage ReadStableIdStorage(string storageDirectory, [Ca
217218
public CodexPage CreateCodexApp(string indexDirectory,
218219
Func<CodexAppOptions, CodexAppOptions> updateOptions = null)
219220
{
220-
var options = new CodexAppOptions();
221-
options = updateOptions?.Invoke(options) ?? options;
221+
var page = getPage();
222+
page.WebViewModelController?.TrackSourceReferenceHtml = true;
223+
return page;
222224

223-
if (PathUtilities.ToUriOrPath(indexDirectory, out var uri, out var path))
225+
CodexPage getPage()
224226
{
225-
// For uri's we need to use paging logic
226-
options.UsePaging = true;
227-
options.ConfigurePaging = options.ConfigurePaging.ApplyBefore(pc => pc with
227+
var options = new CodexAppOptions();
228+
options = updateOptions?.Invoke(options) ?? options;
229+
230+
if (options.CreateSite)
228231
{
229-
CacheLimit = 100_000,
230-
});
231-
}
232+
var arguments = new WebProgramArguments()
233+
{
234+
RootUrl = new Uri("https://codex.test"),
235+
StartUrl = new Uri("https://codex.test"),
236+
IndexSource = new Configuration.IndexSourceLocation()
237+
{
238+
Url = indexDirectory,
239+
}
240+
};
232241

233-
ICodex getCodex()
234-
{
242+
243+
var program = new WebProgramBase(arguments);
244+
245+
program.RunAsync().GetAwaiter().GetResult();
246+
247+
return new CodexPage(program);
248+
}
249+
250+
if (PathUtilities.ToUriOrPath(indexDirectory, out var uri, out var path))
235251
{
236-
LuceneConfiguration configuration;
252+
// For uri's we need to use paging logic
253+
options.UsePaging = true;
254+
options.ConfigurePaging = options.ConfigurePaging.ApplyBefore(pc => pc with
255+
{
256+
CacheLimit = 100_000,
257+
});
258+
}
237259

238-
if (options.UsePaging
239-
|| options.TrackingClient != null)
260+
ICodex getCodex()
261+
{
240262
{
241-
if (options.ValidatePaging)
263+
LuceneConfiguration configuration;
264+
265+
if (options.UsePaging
266+
|| options.TrackingClient != null)
242267
{
243-
var configurePaging = options.ConfigurePaging ?? (pc => pc);
244-
options.ConfigurePaging = pc =>
268+
if (options.ValidatePaging)
245269
{
246-
pc = pc with { Validating = true, ValidatingDirectory = options.ValidatingIndexDirectory ?? indexDirectory };
247-
pc = configurePaging.Invoke(pc);
248-
return pc;
249-
};
250-
}
270+
var configurePaging = options.ConfigurePaging ?? (pc => pc);
271+
options.ConfigurePaging = pc =>
272+
{
273+
pc = pc with { Validating = true, ValidatingDirectory = options.ValidatingIndexDirectory ?? indexDirectory };
274+
pc = configurePaging.Invoke(pc);
275+
return pc;
276+
};
277+
}
251278

252-
if (options.TrackingClient != null)
253-
{
254-
var configurePaging = options.ConfigurePaging ?? (pc => pc);
255-
options.ConfigurePaging = pc =>
279+
if (options.TrackingClient != null)
256280
{
257-
pc = pc with
281+
var configurePaging = options.ConfigurePaging ?? (pc => pc);
282+
options.ConfigurePaging = pc =>
258283
{
259-
UpdateClient = client =>
284+
pc = pc with
260285
{
261-
var trackingClient = options.TrackingClient;
262-
trackingClient.Inner = client.Value;
263-
client.Value = trackingClient;
264-
}
286+
UpdateClient = client =>
287+
{
288+
var trackingClient = options.TrackingClient;
289+
trackingClient.Inner = client.Value;
290+
client.Value = trackingClient;
291+
}
292+
};
293+
pc = configurePaging.Invoke(pc);
294+
295+
return pc;
265296
};
266-
pc = configurePaging.Invoke(pc);
297+
}
267298

268-
return pc;
269-
};
299+
configuration = CreatePagingConfigurationAsync(
300+
indexDirectory,
301+
options.ConfigurePaging).GetAwaiter().GetResult();
302+
}
303+
else
304+
{
305+
configuration = new LuceneConfiguration(indexDirectory);
270306
}
271307

272-
configuration = CreatePagingConfigurationAsync(
273-
indexDirectory,
274-
options.ConfigurePaging).GetAwaiter().GetResult();
275-
}
276-
else
277-
{
278-
configuration = new LuceneConfiguration(indexDirectory);
279-
}
280-
281-
if (options.AddSourceRetriever)
282-
{
283-
configuration.SourceTextRetriever = new HttpClientSourceTextRetriever();
284-
}
308+
if (options.AddSourceRetriever)
309+
{
310+
configuration.SourceTextRetriever = new HttpClientSourceTextRetriever();
311+
}
285312

286-
configuration.DefaultAccessLevel = ObjectModel.RepoAccess.Internal;
313+
configuration.DefaultAccessLevel = ObjectModel.RepoAccess.Internal;
287314

288-
var codex = new LuceneCodex(configuration);
289-
return codex;
315+
var codex = new LuceneCodex(configuration);
316+
return codex;
317+
}
290318
}
291-
}
292319

293-
var codex = getCodex();
320+
var codex = getCodex();
294321

295-
var app = new MainController();
296-
app.Controller = new WebViewModelController(app) { TrackSourceReferenceHtml = true };
297-
app.CodexService = codex;
322+
var app = new MainController();
323+
app.Controller = new WebViewModelController(app);
324+
app.CodexService = codex;
298325

299-
return new CodexPage(codex, app, app.Controller.ViewModel);
326+
return new CodexPage(codex, app, app.Controller.ViewModel);
327+
}
300328
}
301329

302330
public record CodexAppOptions
@@ -305,6 +333,8 @@ public record CodexAppOptions
305333
public string ValidatingIndexDirectory { get; set; }
306334
public bool UsePaging { get; set; }
307335
public bool ValidatePaging { get; set; }
336+
public bool CreateSite { get; set; }
337+
308338
public Func<PagingConfiguration, PagingConfiguration> ConfigurePaging = null;
309339
public TrackingHttpClient TrackingClient { get; set; }
310340
}

‎src/Codex.Sdk/Http/HttpClient.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public interface IInnerHttpClient : IHttpClient
4747

4848
public record QueryAugmentingHttpClientWrapper(IHttpClient Inner, bool FileMode = false, Action<HttpResponseMessage> OnResponse = null) : IHttpClient
4949
{
50-
public Uri BaseAddress { get; set; } = Inner.BaseAddress?.EnsureTrailingSlash();
50+
public Uri BaseAddress { get; } = Inner.BaseAddress?.EnsureTrailingSlash();
5151

5252
public HttpResponseMessage SendMessage(HttpRequestMessage request, CancellationToken token = default)
5353
{

‎src/Codex.Storage/BlobObjectStorage.cs‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
using Azure.Storage.Blobs;
2+
using Azure.Storage.Blobs.Specialized;
23
using Codex.Logging;
34
using Codex.Utilities;
45

56
namespace Codex.Storage;
67

7-
public record BlobObjectStorage(Logger Logger, string BlobSasUrl, string Root = "index") : IObjectStorage
8+
public record BlobObjectStorage(Logger Logger, string BlobSasUrl) : IObjectStorage
89
{
9-
public BlobContainerClient Client { get; private set; }
10+
public BlobClient Folder { get; set; } = new BlobClient(new Uri(BlobSasUrl));
11+
12+
public string FolderPath => PathUtilities.UriCombine(Folder.BlobContainerName, Folder.Name);
1013
public void Dispose()
1114
{
1215
}
@@ -15,10 +18,17 @@ public void Finalize(string message)
1518
{
1619
}
1720

21+
1822
public void Initialize()
1923
{
20-
Client = new BlobContainerClient(new Uri(BlobSasUrl));
21-
24+
var blobUri = new BlobUriBuilder(new Uri(BlobSasUrl));
25+
26+
}
27+
28+
private BlobClient GetBlobClient(string relativePath)
29+
{
30+
var relativePathUrl = PathUtilities.UriCombine(Folder.Name, PathUtilities.AsUrlRelativePath(relativePath, encode: false));
31+
return Folder.GetParentBlobContainerClient().GetBlobClient(relativePathUrl);
2232
}
2333

2434
public Stream Load(string relativePath)
@@ -38,12 +48,11 @@ public async ValueTask<string> WriteAsync(string relativePath, MemoryStream stre
3848
Placeholder.Todo("Avoid uploading if blob is unchanged");
3949
stream.Position = 0;
4050

41-
var relativePathUrl = PathUtilities.UriCombine(Root, PathUtilities.AsUrlRelativePath(relativePath, encode: false));
42-
var blobClient = Client.GetBlobClient(relativePathUrl);
51+
var blobClient = GetBlobClient(relativePath);
4352

4453
var response = await blobClient.UploadAsync(stream, overwrite: true);
4554

46-
return relativePathUrl;
55+
return blobClient.Name;
4756
}
4857

4958
public async Task UploadDirectory(string localDirectory, IEnumerable<string> excludedFiles = null)
@@ -64,14 +73,13 @@ public async Task UploadDirectory(string localDirectory, IEnumerable<string> exc
6473
await Parallel.ForEachAsync(files, async (file, token) =>
6574
{
6675
var relativePath = PathUtilities.GetRelativePath(localDirectory, file);
67-
var relativePathUrl = PathUtilities.UriCombine(Root, PathUtilities.AsUrlRelativePath(relativePath, encode: false));
6876
if (excludesFilesSet.Contains(file))
6977
{
7078
Logger.LogMessage($"Skipping: {relativePath}");
7179
return;
7280
}
7381

74-
var blobClient = Client.GetBlobClient(relativePathUrl);
82+
var blobClient = GetBlobClient(relativePath);
7583

7684

7785
Logger.LogMessage($"Uploading: {relativePath}");

‎src/Codex.Web.Common/ViewModels/CodexPage.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using Codex.Sdk.Search;
33
using Codex.Storage;
44
using Codex.Utilities;
5+
using Codex.Web.Common;
56
using Codex.Web.Mvc.Rendering;
67
using Codex.Web.Wasm;
78

@@ -19,6 +20,12 @@ public CodexPage(ICodex codex, MainController app)
1920
{
2021
}
2122

23+
public CodexPage(WebProgramBase site)
24+
: this(site.App.CodexService, site.App)
25+
{
26+
Site = site;
27+
}
28+
2229
public async Task<INavigateItem> SelectLeftItem(int index)
2330
{
2431
var item = View.LeftPane.Content.GetItems().ElementAt(index);
@@ -28,6 +35,10 @@ public async Task<INavigateItem> SelectLeftItem(int index)
2835

2936
public IStableIdStorage StableIdStorage { get; set; }
3037

38+
public WebViewModelController? WebViewModelController => App.Controller as WebViewModelController;
39+
40+
public WebProgramBase Site { get; init; }
41+
3142
public Task Search(string text) => App.SearchTextChanged(text);
3243

3344
public IReadOnlyList<INavigateItem> LeftItems => View.LeftPane.Content?.ItemList ?? (IReadOnlyList<INavigateItem>)Array.Empty<INavigateItem>();

‎src/Codex.Web.Common/WebProgramBase.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ public class WebProgramBase(WebProgramArguments args) : CodexProgramBase, IRepos
2727
public async Task RunAsync(ViewModelAddress? address = null)
2828
{
2929
Controller.RepositoryIndexer = this;
30-
31-
var client = new QueryAugmentingHttpClientWrapper(GetClient(default));
32-
client.BaseAddress = GetBaseAddress();
30+
var innerClient = GetClient(default);
31+
innerClient.BaseAddress = GetBaseAddress().EnsureTrailingSlash();
32+
var client = new QueryAugmentingHttpClientWrapper(innerClient);
33+
Client = client;
3334
SdkFeatures.HttpClient = client;
3435
SdkFeatures.GetClient = GetClient;
3536

36-
var codex = await GetCodexAsync(address);
37+
App.CodexService ??= await GetCodexAsync(address);
3738
App.Controller = Controller;
38-
App.CodexService = codex;
3939
//if (address != null)
4040
//{
4141
// await address.NavigateAsync(App, ViewModelAddress.InferMode.Startup);
@@ -44,7 +44,7 @@ public async Task RunAsync(ViewModelAddress? address = null)
4444

4545
public CodexPage GetPage()
4646
{
47-
return new CodexPage(App.CodexService, App);
47+
return new CodexPage(this);
4848
}
4949

5050
private Uri GetBaseAddress()

0 commit comments

Comments
 (0)