Skip to content

Commit 793bbb8

Browse files
FosolFosol
committed
Created Elastic Cloud Indexer Service (bcgov#2470)
* Created Cloud Indexer Service Remove sonar scan Update DevOps * Remove readme * Review comments
1 parent bea4879 commit 793bbb8

File tree

63 files changed

+1602
-232
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+1602
-232
lines changed

‎.github/workflows/net-libs.yml‎

Lines changed: 66 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -115,69 +115,69 @@ jobs:
115115
# Specify whether or not CI build should fail if Codecov runs into an error during upload
116116
fail_ci_if_error: true
117117

118-
- name: SonarScanner for .NET 9 with pull request decoration support
119-
id: scan
120-
uses: highbyte/sonarscan-dotnet@2.0
121-
if: ${{ false }}
122-
# if: github.event_name == 'push'
123-
env:
124-
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
125-
with:
126-
dotnetBuildArguments: ${{env.working-directory}}
127-
dotnetTestArguments: ${{env.working-directory}} --logger trx --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
128-
# Optional extra command arguments the the SonarScanner 'begin' command
129-
sonarBeginArguments: /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" -d:sonar.cs.vstest.reportsPaths="**/TestResults/*.trx" -d:sonar.qualitygate.wait=true
130-
# The key of the SonarQube project
131-
sonarProjectKey: ${{secrets.PROJECT_KEY_API}}
132-
# The name of the SonarQube project
133-
sonarProjectName: PIMS-API
134-
# The SonarQube server URL. For SonarCloud, skip this setting.
135-
sonarHostname: ${{secrets.SONAR_URL}}
136-
137-
- name: Find Comment
138-
if: ${{ false }}
139-
# if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
140-
uses: peter-evans/find-comment@v1
141-
id: fc
142-
with:
143-
issue-number: ${{ github.event.pull_request.number }}
144-
comment-author: "github-actions[bot]"
145-
body-includes: QUALITY GATE STATUS FOR .NET 9
146-
147-
- name: Check Quality Gate and Create Comment
148-
if: ${{ false }}
149-
# if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id == ''
150-
uses: peter-evans/create-or-update-comment@v1
151-
env:
152-
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
153-
SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
154-
PROJECT_KEY: ${{secrets.PROJECT_KEY_API}}
155-
with:
156-
issue-number: ${{ github.event.pull_request.number }}
157-
body: |
158-
QUALITY GATE STATUS FOR .NET 9: FAILED.
159-
160-
[View and resolve details on][1]
161-
162-
[1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
163-
reactions: confused
164-
165-
- name: Check Quality Gate and Update Comment
166-
if: ${{ false }}
167-
# if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id != '' && github.event.pull_request.head.repo.full_name == github.repository
168-
uses: peter-evans/create-or-update-comment@v1
169-
env:
170-
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
171-
SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
172-
PROJECT_KEY: ${{secrets.PROJECT_KEY_APP}}
173-
with:
174-
comment-id: ${{ steps.fc.outputs.comment-id }}
175-
issue-number: ${{ github.event.pull_request.number }}
176-
body: |
177-
QUALITY GATE STATUS FOR .NET 9: FAILED.
178-
179-
[View and resolve details on][1]
180-
181-
[1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
182-
edit-mode: replace
183-
reactions: eyes
118+
# - name: SonarScanner for .NET 9 with pull request decoration support
119+
# id: scan
120+
# uses: highbyte/sonarscan-dotnet@2.0
121+
# if: ${{ false }}
122+
# # if: github.event_name == 'push'
123+
# env:
124+
# SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
125+
# with:
126+
# dotnetBuildArguments: ${{env.working-directory}}
127+
# dotnetTestArguments: ${{env.working-directory}} --logger trx --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
128+
# # Optional extra command arguments the the SonarScanner 'begin' command
129+
# sonarBeginArguments: /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" -d:sonar.cs.vstest.reportsPaths="**/TestResults/*.trx" -d:sonar.qualitygate.wait=true
130+
# # The key of the SonarQube project
131+
# sonarProjectKey: ${{secrets.PROJECT_KEY_API}}
132+
# # The name of the SonarQube project
133+
# sonarProjectName: TNO-API
134+
# # The SonarQube server URL. For SonarCloud, skip this setting.
135+
# sonarHostname: ${{secrets.SONAR_URL}}
136+
137+
# - name: Find Comment
138+
# if: ${{ false }}
139+
# # if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
140+
# uses: peter-evans/find-comment@v1
141+
# id: fc
142+
# with:
143+
# issue-number: ${{ github.event.pull_request.number }}
144+
# comment-author: "github-actions[bot]"
145+
# body-includes: QUALITY GATE STATUS FOR .NET 9
146+
147+
# - name: Check Quality Gate and Create Comment
148+
# if: ${{ false }}
149+
# # if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id == ''
150+
# uses: peter-evans/create-or-update-comment@v1
151+
# env:
152+
# SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
153+
# SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
154+
# PROJECT_KEY: ${{secrets.PROJECT_KEY_API}}
155+
# with:
156+
# issue-number: ${{ github.event.pull_request.number }}
157+
# body: |
158+
# QUALITY GATE STATUS FOR .NET 9: FAILED.
159+
160+
# [View and resolve details on][1]
161+
162+
# [1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
163+
# reactions: confused
164+
165+
# - name: Check Quality Gate and Update Comment
166+
# if: ${{ false }}
167+
# # if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id != '' && github.event.pull_request.head.repo.full_name == github.repository
168+
# uses: peter-evans/create-or-update-comment@v1
169+
# env:
170+
# SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
171+
# SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
172+
# PROJECT_KEY: ${{secrets.PROJECT_KEY_APP}}
173+
# with:
174+
# comment-id: ${{ steps.fc.outputs.comment-id }}
175+
# issue-number: ${{ github.event.pull_request.number }}
176+
# body: |
177+
# QUALITY GATE STATUS FOR .NET 9: FAILED.
178+
179+
# [View and resolve details on][1]
180+
181+
# [1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
182+
# edit-mode: replace
183+
# reactions: eyes

‎.vscode/launch.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,23 @@
309309
"stopAtEntry": false,
310310
"envFile": "${workspaceFolder}/tools/elastic/migration/.env"
311311
},
312+
{
313+
// Use IntelliSense to find out which attributes exist for C# debugging
314+
// Use hover for the description of the existing attributes
315+
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
316+
"name": "Run Elastic Indexer Service",
317+
"type": "coreclr",
318+
"request": "launch",
319+
"preLaunchTask": "build-elastic-indexer",
320+
// If you have changed target frameworks, make sure to update the program path.
321+
"program": "${workspaceFolder}/tools/indexer/bin/Debug/net9.0/TNO.Tools.ElasticIndexer.dll",
322+
"args": [],
323+
"cwd": "${workspaceFolder}/tools/indexer",
324+
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
325+
"console": "internalConsole",
326+
"stopAtEntry": false,
327+
"envFile": "${workspaceFolder}/tools/indexer/.env"
328+
},
312329
{
313330
"name": ".NET Core Attach",
314331
"type": "coreclr",

‎.vscode/tasks.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -649,5 +649,42 @@
649649
],
650650
"problemMatcher": "$msCompile"
651651
}
652+
,
653+
{
654+
"label": "build-elastic-indexer",
655+
"command": "dotnet",
656+
"type": "process",
657+
"args": [
658+
"build",
659+
"${workspaceFolder}/tools/indexer/TNO.Tools.ElasticIndexer.csproj",
660+
"/property:GenerateFullPaths=true",
661+
"/consoleloggerparameters:NoSummary"
662+
],
663+
"problemMatcher": "$msCompile"
664+
},
665+
{
666+
"label": "publish-elastic-indexer",
667+
"command": "dotnet",
668+
"type": "process",
669+
"args": [
670+
"publish",
671+
"${workspaceFolder}/tools/indexer/TNO.Tools.ElasticIndexer.csproj",
672+
"/property:GenerateFullPaths=true",
673+
"/consoleloggerparameters:NoSummary"
674+
],
675+
"problemMatcher": "$msCompile"
676+
},
677+
{
678+
"label": "watch-elastic-indexer",
679+
"command": "dotnet",
680+
"type": "process",
681+
"args": [
682+
"watch",
683+
"run",
684+
"--project",
685+
"${workspaceFolder}/tools/indexer/TNO.Tools.ElasticIndexer.csproj",
686+
],
687+
"problemMatcher": "$msCompile"
688+
}
652689
]
653690
}

‎api/net/Areas/Admin/Controllers/ChartTemplateController.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ public async Task<IActionResult> PreviewJsonAsync(ChartPreviewRequestModel model
175175
if (filter != null)
176176
{
177177
var filterSettings = JsonSerializer.Deserialize<API.Models.Settings.FilterSettingsModel>(filter.Settings);
178-
var index = filterSettings?.SearchUnpublished == true ? _elasticOptions.UnpublishedIndex : _elasticOptions.PublishedIndex;
178+
var index = filterSettings?.SearchUnpublished == true ? _elasticOptions.ContentIndex : _elasticOptions.PublishedIndex;
179179
content = content.AppendRange(await _reportHelper.FindContentAsync(filter.Query, index));
180180
}
181181
}

‎api/net/Areas/Editor/Controllers/ContentController.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ public IActionResult FindWithDatabase()
155155
[SwaggerOperation(Tags = new[] { "Content" })]
156156
public async Task<IActionResult> FindWithElasticsearchAsync([FromBody] JsonDocument filter, [FromQuery] bool includeUnpublishedContent = false)
157157
{
158-
var result = await _contentService.FindWithElasticsearchAsync(includeUnpublishedContent ? _elasticOptions.UnpublishedIndex : _elasticOptions.PublishedIndex, filter);
158+
var result = await _contentService.FindWithElasticsearchAsync(includeUnpublishedContent ? _elasticOptions.ContentIndex : _elasticOptions.PublishedIndex, filter);
159159
return new JsonResult(result);
160160
}
161161

‎api/net/Areas/Subscriber/Controllers/ContentController.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ public async Task<IActionResult> FindWithElasticsearchAsync([FromBody] JsonDocum
121121
filter = filter.AddExcludeSources(user.SourcesManyToMany.Select(s => s.SourceId));
122122
filter = filter.AddExcludeMediaTypes(user.MediaTypesManyToMany.Select(s => s.MediaTypeId));
123123

124-
var result = await _contentService.FindWithElasticsearchAsync(includeUnpublishedContent ? _elasticOptions.UnpublishedIndex : _elasticOptions.PublishedIndex, filter);
124+
var result = await _contentService.FindWithElasticsearchAsync(includeUnpublishedContent ? _elasticOptions.ContentIndex : _elasticOptions.PublishedIndex, filter);
125125
return new JsonResult(result);
126126
}
127127

‎api/net/appsettings.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@
9090
},
9191
"Elastic": {
9292
"Url": "http://elastic:9200",
93-
"UnpublishedIndex": "unpublished_content",
93+
"ContentIndex": "unpublished_content",
9494
"PublishedIndex": "content"
9595
},
9696
"SignalR": {

‎docker-compose.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,18 @@ services:
122122
# retries: 3
123123
# start_period: 15s
124124

125+
126+
indexer:
127+
profiles:
128+
- indexer
129+
build:
130+
context: ./
131+
dockerfile: tools/indexer/Dockerfile
132+
env_file:
133+
- tools/indexer/.env
134+
networks:
135+
- tno
136+
125137
####################### API #######################
126138
api:
127139
image: tno:api

‎libs/net/dal/Extensions/ServiceCollectionExtensions.cs‎

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,45 @@ public static IServiceCollection AddTNOContext(this IServiceCollection services,
5656
return services;
5757
}
5858

59+
/// <summary>
60+
/// Add a PostgreSQL DbContext to the service collection.
61+
/// </summary>
62+
/// <param name="services"></param>
63+
/// <param name="connectionString"></param>
64+
/// <param name="env"></param>
65+
/// <returns></returns>
66+
/// <exception cref="ArgumentException"></exception>
67+
public static IServiceCollection AddSingletonTNOContext(this IServiceCollection services, string connectionString, IHostEnvironment env)
68+
{
69+
if (String.IsNullOrWhiteSpace(connectionString)) throw new ArgumentException("Argument is required and cannot be null, empty or whitespace.", nameof(connectionString));
70+
71+
var dataSourceBuilder = new NpgsqlDataSourceBuilder(connectionString);
72+
dataSourceBuilder.EnableDynamicJson();
73+
var dataSource = dataSourceBuilder.Build();
74+
services.AddDbContext<TNOContext>(options =>
75+
{
76+
var db = options.UseNpgsql(dataSource, options =>
77+
{
78+
options.CommandTimeout((int)TimeSpan.FromMinutes(5).TotalSeconds);
79+
});
80+
options.EnableSensitiveDataLogging(env.IsDevelopment());
81+
options.EnableDetailedErrors(env.IsDevelopment());
82+
if (env.IsDevelopment())
83+
{
84+
var debugLoggerFactory = LoggerFactory
85+
.Create(builder =>
86+
{
87+
builder
88+
.AddConsole()
89+
.AddDebug();
90+
});
91+
db.UseLoggerFactory(debugLoggerFactory);
92+
}
93+
}, ServiceLifetime.Singleton);
94+
95+
return services;
96+
}
97+
5998
/// <summary>
6099
/// Add a PostgreSQL DbContext to the service collection.
61100
/// </summary>
@@ -73,6 +112,23 @@ public static IServiceCollection AddTNOContext(this IServiceCollection services,
73112
return services.AddTNOContext(postgresBuilder.ConnectionString, env);
74113
}
75114

115+
/// <summary>
116+
/// Add a PostgreSQL DbContext to the service collection.
117+
/// </summary>
118+
/// <param name="services"></param>
119+
/// <param name="config"></param>
120+
/// <param name="env"></param>
121+
/// <returns></returns>
122+
public static IServiceCollection AddSingletonTNOContext(this IServiceCollection services, IConfiguration config, IHostEnvironment env)
123+
{
124+
var postgresBuilder = new NpgsqlConnectionStringBuilder(config["ConnectionStrings:TNO"])
125+
{
126+
Username = config["DB_POSTGRES_USERNAME"],
127+
Password = config["DB_POSTGRES_PASSWORD"]
128+
};
129+
return services.AddSingletonTNOContext(postgresBuilder.ConnectionString, env);
130+
}
131+
76132
/// <summary>
77133
/// Add a PostgreSQL DbContext to the service collection and all the related services.
78134
/// </summary>
@@ -84,7 +140,7 @@ public static IServiceCollection AddTNOContext(this IServiceCollection services,
84140
/// <exception cref="InvalidOperationException"></exception>
85141
public static IServiceCollection AddTNOServices(this IServiceCollection services, IConfiguration config, IHostEnvironment env)
86142
{
87-
if (config == null) throw new ArgumentNullException(nameof(config));
143+
ArgumentNullException.ThrowIfNull(config);
88144

89145
services.AddTNOContext(config, env)
90146
.AddStorageConfig(config)
@@ -97,8 +153,39 @@ public static IServiceCollection AddTNOServices(this IServiceCollection services
97153
var tnoServiceTypes = assembly.GetTypes().Where(t => !t.IsAbstract && t.IsClass && t.GetInterfaces().Any(i => i.Name.Equals(type.Name)));
98154
foreach (var serviceType in tnoServiceTypes)
99155
{
100-
var sinterface = serviceType.GetInterface($"I{serviceType.Name}") ?? throw new InvalidOperationException($"Service type '{serviceType.Name}' is missing its interface.");
101-
services.AddScoped(sinterface, serviceType);
156+
var sInterface = serviceType.GetInterface($"I{serviceType.Name}") ?? throw new InvalidOperationException($"Service type '{serviceType.Name}' is missing its interface.");
157+
services.AddScoped(sInterface, serviceType);
158+
}
159+
160+
return services;
161+
}
162+
163+
/// <summary>
164+
/// Add a PostgreSQL DbContext to the service collection and all the related services.
165+
/// </summary>
166+
/// <param name="services"></param>
167+
/// <param name="config"></param>
168+
/// <param name="env"></param>
169+
/// <returns></returns>
170+
/// <exception cref="ArgumentNullException"></exception>
171+
/// <exception cref="InvalidOperationException"></exception>
172+
public static IServiceCollection AddSingletonTNOServices(this IServiceCollection services, IConfiguration config, IHostEnvironment env)
173+
{
174+
ArgumentNullException.ThrowIfNull(config);
175+
176+
services.AddSingletonTNOContext(config, env)
177+
.AddStorageConfig(config)
178+
.AddSingletonElastic(config, env)
179+
.AddSingleton<IElasticsearchService, ElasticsearchService>();
180+
181+
// Find all the configuration classes.
182+
var assembly = typeof(BaseService).Assembly;
183+
var type = typeof(IBaseService);
184+
var tnoServiceTypes = assembly.GetTypes().Where(t => !t.IsAbstract && t.IsClass && t.GetInterfaces().Any(i => i.Name.Equals(type.Name)));
185+
foreach (var serviceType in tnoServiceTypes)
186+
{
187+
var sInterface = serviceType.GetInterface($"I{serviceType.Name}") ?? throw new InvalidOperationException($"Service type '{serviceType.Name}' is missing its interface.");
188+
services.AddSingleton(sInterface, serviceType);
102189
}
103190

104191
return services;

‎libs/net/dal/Services/NotificationService.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ public async Task<int> Unsubscribe(int userId)
281281

282282
var ownerId = requestorId ?? notification.OwnerId; // TODO: Handle users generating instances for a notifications they do not own.
283283

284-
var defaultIndex = filter.SearchUnpublished ? _elasticOptions.UnpublishedIndex : _elasticOptions.PublishedIndex;
284+
var defaultIndex = filter.SearchUnpublished ? _elasticOptions.ContentIndex : _elasticOptions.PublishedIndex;
285285
var query = notification.Query;
286286
var topStoryLastRunOnSetting = _settingService.FindByName(TopStoryLastRunOn);
287287

0 commit comments

Comments
 (0)