Skip to content

Commit f0921d6

Browse files
JVickery-TBSJVickery-TBS
committed
cont.
1 parent cae6b95 commit f0921d6

File tree

3 files changed

+45
-39
lines changed

3 files changed

+45
-39
lines changed

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ This plugin hooks into the creation and insertion of DataStore tables and record
77

88
## Requirements
99

10+
SOLR 9+ running in Stand Alone mode. Currently, this plugin does NOT support Cloud Mode SOLR.
11+
1012
https://github.com/ckan/ckan/pull/8684
1113

1214
Compatibility with core CKAN versions:

‎ckanext/datastore_search/backend/solr.py‎

Lines changed: 41 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
PSQL_TO_SOLR_WILCARD_MATCH = re.compile('^_?|_?$')
2323

2424
log = getLogger(__name__)
25+
DEBUG = config.get('debug', False)
2526

2627

2728
class DatastoreSolrBackend(DatastoreSearchBackend):
@@ -105,7 +106,7 @@ def _make_or_create_connection(self, resource_id: str) -> Optional[pysolr.Solr]:
105106
create_dict = {
106107
'resource_id': resource_id,
107108
'fields': [f for f in ds_result['fields'] if
108-
f['id'] not in self.default_solr_fields]}
109+
f['id'] not in self.default_search_fields]}
109110
self.create(self._get_site_context(), create_dict)
110111
conn = self._make_connection(resource_id)
111112
if conn:
@@ -159,7 +160,7 @@ def reindex(self,
159160
endpoint=f'cores/{core_name}/reload')
160161
if 'error' in resp:
161162
raise DatastoreSearchException(
162-
errmsg if not config.get('debug')
163+
errmsg if not DEBUG
163164
else resp['error'].get('msg', errmsg)[:MAX_ERR_LEN])
164165
log.debug('Reloaded SOLR Core for DataStore Resource %s' % resource_id)
165166

@@ -170,7 +171,7 @@ def reindex(self,
170171
ds_total = ds_result['total']
171172
ds_field_ids = []
172173
for ds_field in ds_result.get('fields', []):
173-
if ds_field['id'] not in self.default_solr_fields:
174+
if ds_field['id'] not in self.default_search_fields:
174175
ds_field_ids.append(ds_field['id'])
175176
ds_columns = '_id,' + ','.join([identifier(c) for c in ds_field_ids])
176177

@@ -218,22 +219,25 @@ def reindex(self,
218219
continue
219220
try:
220221
conn.add(docs=[r], commit=False)
221-
log.debug('Indexed DataStore record _id=%s for Resource %s' %
222-
(r['_id'], resource_id))
222+
if DEBUG:
223+
log.debug('Indexed DataStore record _id=%s for Resource %s' %
224+
(r['_id'], resource_id))
223225
except pysolr.SolrError as e:
224226
raise DatastoreSearchException(
225-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
227+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
226228
offset += 1000
227229
orphan_ids = set(indexed_ids) - set(existing_ids)
228230
for orphan_id in orphan_ids:
229231
try:
230232
conn.delete(q='_id:%s' % orphan_id, commit=False)
231-
log.debug('Unindexed DataStore record _id=%s for Resource %s' %
232-
(orphan_id, resource_id))
233+
if DEBUG:
234+
log.debug('Unindexed DataStore record _id=%s for Resource %s' %
235+
(orphan_id, resource_id))
233236
except pysolr.SolrError as e:
234237
raise DatastoreSearchException(
235-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
238+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
236239
conn.commit(waitSearcher=False)
240+
log.debug('Reindexed SOLR Core for DataStore Resource %s' % resource_id)
237241

238242
def _check_counts(self,
239243
resource_id: str,
@@ -267,6 +271,12 @@ def create(self,
267271
core_name = f'{self.prefix}{resource_id}'
268272
conn = self._make_connection(resource_id) if not connection else connection
269273
if not conn:
274+
# FIXME: using configSet in API does not copy the configSet
275+
# into the core conf directory. We need to send some type
276+
# of signal to the SOLR server so it can run
277+
# solr create -c core_name -d configsets/datastore_resource
278+
# then does SOLR server need to send a signal back?
279+
# or can we just keep retrying a couple of times??
270280
errmsg = _('Could not create SOLR core %s') % core_name
271281
req_body = {'create': [{'name': core_name,
272282
'configSet': 'datastore_resource'}]}
@@ -275,7 +285,7 @@ def create(self,
275285
body=req_body)
276286
if 'error' in resp:
277287
raise DatastoreSearchException(
278-
errmsg if not config.get('debug')
288+
errmsg if not DEBUG
279289
else resp['error'].get('msg', errmsg)[:MAX_ERR_LEN])
280290
log.debug('Created SOLR Core for DataStore Resource %s' % resource_id)
281291
conn = self._make_connection(resource_id)
@@ -288,18 +298,18 @@ def create(self,
288298
method='GET', path='schema/fields'))['fields']
289299
except pysolr.SolrError as e:
290300
raise DatastoreSearchException(
291-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
301+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
292302
keyed_solr_fields = {}
293303
for solr_field in solr_fields:
294-
if solr_field['name'] in self.default_solr_fields:
304+
if solr_field['name'] in self.default_search_fields:
295305
continue
296306
keyed_solr_fields[solr_field['name']] = solr_field
297307
ds_field_ids = []
298308
new_fields = []
299309
updated_fields = []
300310
remove_fields = []
301311
for ds_field in data_dict.get('fields', []):
302-
if ds_field['id'] not in self.default_solr_fields:
312+
if ds_field['id'] not in self.default_search_fields:
303313
ds_field_ids.append(ds_field['id'])
304314
if ds_field['id'] not in keyed_solr_fields:
305315
new_fields.append({
@@ -325,10 +335,10 @@ def create(self,
325335
headers={'Content-Type': 'application/json'}))
326336
except pysolr.SolrError as e:
327337
raise DatastoreSearchException(
328-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
338+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
329339
if 'error' in resp:
330340
raise DatastoreSearchException(
331-
errmsg if not config.get('debug')
341+
errmsg if not DEBUG
332342
else resp['error'].get('msg', errmsg)[:MAX_ERR_LEN])
333343
log.debug('Added SOLR Field %s for DataStore Resource %s' %
334344
(f['name'], resource_id))
@@ -343,10 +353,10 @@ def create(self,
343353
headers={'Content-Type': 'application/json'}))
344354
except pysolr.SolrError as e:
345355
raise DatastoreSearchException(
346-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
356+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
347357
if 'error' in resp:
348358
raise DatastoreSearchException(
349-
errmsg if not config.get('debug')
359+
errmsg if not DEBUG
350360
else resp['error'].get('msg', errmsg)[:MAX_ERR_LEN])
351361
log.debug('Modified SOLR Field %s for DataStore Resource %s' %
352362
(f['name'], resource_id))
@@ -361,10 +371,10 @@ def create(self,
361371
headers={'Content-Type': 'application/json'}))
362372
except pysolr.SolrError as e:
363373
raise DatastoreSearchException(
364-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
374+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
365375
if 'error' in resp:
366376
raise DatastoreSearchException(
367-
errmsg if not config.get('debug')
377+
errmsg if not DEBUG
368378
else resp['error'].get('msg', errmsg)[:MAX_ERR_LEN])
369379
log.debug('Removed SOLR Field %s for DataStore Resource %s' %
370380
(f['name'], resource_id))
@@ -392,12 +402,13 @@ def upsert(self,
392402
for r in data_dict['records']:
393403
try:
394404
conn.add(docs=[r], commit=False)
395-
log.debug('Indexed DataStore record _id=%s for Resource %s' %
396-
(r['_id'], resource_id))
405+
if DEBUG:
406+
log.debug('Indexed DataStore record _id=%s for Resource %s' %
407+
(r['_id'], resource_id))
397408
except pysolr.SolrError as e:
398409
errmsg = _('Failed to index records for %s' % core_name)
399410
raise DatastoreSearchException(
400-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
411+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
401412
conn.commit(waitSearcher=False)
402413

403414
self._check_counts(resource_id, connection=conn)
@@ -428,6 +439,7 @@ def search(self,
428439
'*',
429440
value.replace(':*', ''))))
430441
if query and isinstance(query, str):
442+
# FIXME: solve the query of all _text_ field
431443
q = '*%s*' % re.sub(PSQL_TO_SOLR_WILCARD_MATCH,
432444
'*',
433445
query.replace(':*', ''))
@@ -452,7 +464,7 @@ def search(self,
452464
except pysolr.SolrError as e:
453465
errmsg = _('Failed to query records for resource %s' % resource_id)
454466
raise DatastoreSearchException(
455-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
467+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
456468

457469
return results.docs
458470

@@ -474,14 +486,14 @@ def delete(self,
474486
log.debug('Unindexed all DataStore records for Resource %s' % resource_id)
475487
except pysolr.SolrError as e:
476488
raise DatastoreSearchException(
477-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
489+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
478490
conn.commit(waitSearcher=False)
479491
if data_dict.get('filters') is None:
480492
resp = self._send_api_request(
481493
method='POST', endpoint=f'cores/{core_name}/unload')
482494
if 'error' in resp:
483495
raise DatastoreSearchException(
484-
errmsg if not config.get('debug')
496+
errmsg if not DEBUG
485497
else resp['error'].get('msg', errmsg)[:MAX_ERR_LEN])
486498
log.debug('Unloaded SOLR Core for DataStore Resource %s' % resource_id)
487499
return
@@ -491,11 +503,12 @@ def delete(self,
491503
(record['_id'], core_name)
492504
try:
493505
conn.delete(q='_id:%s' % record['_id'], commit=False)
494-
log.debug('Unindexed DataStore record _id=%s for Resource %s' %
495-
(record['_id'], resource_id))
506+
if DEBUG:
507+
log.debug('Unindexed DataStore record _id=%s for Resource %s' %
508+
(record['_id'], resource_id))
496509
except pysolr.SolrError as e:
497510
raise DatastoreSearchException(
498-
errmsg if not config.get('debug') else e.args[0][:MAX_ERR_LEN])
511+
errmsg if not DEBUG else e.args[0][:MAX_ERR_LEN])
499512
conn.commit(waitSearcher=False)
500513

501514
self._check_counts(resource_id, connection=conn)

‎ckanext/datastore_search/plugin.py‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,9 @@ class DataStoreSearchPlugin(plugins.SingletonPlugin):
2222
plugins.implements(plugins.IConfigurer)
2323
plugins.implements(plugins.IActions)
2424
plugins.implements(IDatastoreSearchBackend, inherit=True)
25+
plugins.implements(IDataPusher, inherit=True)
2526
if HAS_XLOADER:
26-
try:
27-
plugins.toolkit.get_action('xloader_submit')
28-
plugins.implements(IXloader, inherit=True)
29-
except KeyError:
30-
pass
31-
else:
32-
try:
33-
plugins.toolkit.get_action('datapusher_submit')
34-
plugins.implements(IDataPusher, inherit=True)
35-
except KeyError:
36-
pass
27+
plugins.implements(IXloader, inherit=True)
3728

3829
# IDatastoreSearchBackend
3930
def register_backends(self):

0 commit comments

Comments
 (0)