diff --git a/bom/application/pom.xml b/bom/application/pom.xml index eaeeda916d95..ad8f2200e10e 100644 --- a/bom/application/pom.xml +++ b/bom/application/pom.xml @@ -26,6 +26,7 @@ 2.28 22.3.3 1.13.10 + 3.3.0 @@ -839,6 +840,41 @@ 4.4.6 + + + org.opensearch.client + opensearch-java + ${opensearch.version} + + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5 + + + + + + org.apache.httpcomponents.client5 + httpclient5 + 5.5 + + + + org.apache.httpcomponents.core5 + httpcore5 + 5.3.4 + + + + org.apache.httpcomponents.core5 + httpcore5-h2 + 5.3.4 + + org.elasticsearch.client elasticsearch-rest-high-level-client diff --git a/dotCMS/pom.xml b/dotCMS/pom.xml index 2c1acbf8eee4..f1ebaa643510 100644 --- a/dotCMS/pom.xml +++ b/dotCMS/pom.xml @@ -727,6 +727,36 @@ jedis + + org.opensearch.client + opensearch-java + + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5 + + + + + + org.apache.httpcomponents.core5 + httpcore5 + + + + org.apache.httpcomponents.core5 + httpcore5-h2 + + + + org.apache.httpcomponents.client5 + httpclient5 + + org.elasticsearch.client elasticsearch-rest-high-level-client diff --git a/dotCMS/src/enterprise/java/com/dotcms/enterprise/priv/ESSearchAPIImpl.java b/dotCMS/src/enterprise/java/com/dotcms/enterprise/priv/ESSearchAPIImpl.java index 69806fdad2bb..99f66c9152c0 100644 --- a/dotCMS/src/enterprise/java/com/dotcms/enterprise/priv/ESSearchAPIImpl.java +++ b/dotCMS/src/enterprise/java/com/dotcms/enterprise/priv/ESSearchAPIImpl.java @@ -9,9 +9,11 @@ package com.dotcms.enterprise.priv; +import static com.dotcms.content.elasticsearch.business.ESIndexAPI.INDEX_OPERATIONS_TIMEOUT_IN_MS; + import com.dotcms.content.elasticsearch.business.ESContentFactoryImpl; import com.dotcms.content.elasticsearch.business.ESSearchResults; -import com.dotcms.content.elasticsearch.business.IndiciesInfo; +import com.dotcms.content.elasticsearch.business.IndicesInfo; import com.dotcms.content.elasticsearch.util.RestHighLevelClientProvider; import com.dotcms.enterprise.ESSeachAPI; import com.dotcms.enterprise.priv.util.SearchSourceBuilderUtil; @@ -29,6 +31,10 @@ import com.dotmarketing.util.json.JSONException; import com.dotmarketing.util.json.JSONObject; import com.liferay.portal.model.User; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -37,13 +43,6 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static com.dotcms.content.elasticsearch.business.ESIndexAPI.INDEX_OPERATIONS_TIMEOUT_IN_MS; - /** * Implementation class for the {@link ESSeachAPI}. * @@ -200,9 +199,9 @@ private SearchResponse esSearchRaw(JSONObject jsonObject, boolean live, User use throws DotSecurityException, DotDataException { String indexToHit; - IndiciesInfo info; + IndicesInfo info; try { - info = APILocator.getIndiciesAPI().loadIndicies(); + info = APILocator.getIndiciesAPI().loadLegacyIndices(); if (live) { indexToHit = info.getLive(); } else { diff --git a/dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/sitesearch/ESSiteSearchAPI.java b/dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/sitesearch/ESSiteSearchAPI.java index 44100ee0491d..83b2a235e941 100644 --- a/dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/sitesearch/ESSiteSearchAPI.java +++ b/dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/sitesearch/ESSiteSearchAPI.java @@ -73,17 +73,17 @@ public class ESSiteSearchAPI implements SiteSearchAPI{ private final ESIndexAPI indexApi; private final ESMappingAPIImpl mappingAPI; - private final IndiciesAPI indiciesAPI; + private final IndicesAPI indicesAPI; private ArrayList list; private int indexPosition; @VisibleForTesting public ESSiteSearchAPI(final ESIndexAPI indexApi, final ESMappingAPIImpl mappingAPI, - final IndiciesAPI indiciesAPI) { + final IndicesAPI indicesAPI) { this.indexApi = indexApi; this.mappingAPI = mappingAPI; - this.indiciesAPI = indiciesAPI; + this.indicesAPI = indicesAPI; } public ESSiteSearchAPI() { @@ -122,7 +122,7 @@ private void setDefaultToSpecificPosition(final List list, final int ind try { //search the default site search index - final String defaultIndice = indiciesAPI.loadIndicies().getSiteSearch(); + final String defaultIndice = indicesAPI.loadLegacyIndices().getSiteSearch(); if (defaultIndice != null && !defaultIndice.isEmpty() && !list.isEmpty() ){ final int index = list.indexOf(defaultIndice); //change the element defaultIndex to the first position of the arraylist if it is not yet @@ -170,7 +170,7 @@ public SiteSearchResults search(String query, int start, int rows) throws Elasti try{ - results = search(indiciesAPI.loadIndicies().getSiteSearch(), query, start, rows); + results = search(indicesAPI.loadLegacyIndices().getSiteSearch(), query, start, rows); } @@ -210,7 +210,7 @@ public SiteSearchResults search(String indexName, String query, int offset, int SearchResponse resp = null; try { if(indexName ==null){ - indexName = indiciesAPI.loadIndicies().getSiteSearch(); + indexName = indicesAPI.loadLegacyIndices().getSiteSearch(); } if(!IndexType.SITE_SEARCH.is(indexName)){ throw new ElasticsearchException(indexName + " is not a sitesearch index"); @@ -317,7 +317,7 @@ public SiteSearchResults search(String indexName, String query, int offset, int */ @Override public boolean isDefaultIndex(final String indexName) throws DotDataException { - return indexName.equals(indiciesAPI.loadIndicies().getSiteSearch()); + return indexName.equals(indicesAPI.loadLegacyIndices().getSiteSearch()); } @Override @@ -325,14 +325,14 @@ public void activateIndex(String indexName) throws DotDataException { if(LicenseUtil.getLevel() < LicenseLevel.STANDARD.level) return; - final IndiciesInfo info = indiciesAPI.loadIndicies(); - final IndiciesInfo.Builder builder = IndiciesInfo.Builder.copy(info); + final IndicesInfo info = indicesAPI.loadLegacyIndices(); + final LegacyIndicesInfo.Builder builder = LegacyIndicesInfo.Builder.copy(info); if(IndexType.SITE_SEARCH.is(indexName)) { builder.setSiteSearch(indexName); } - indiciesAPI.point(builder.build()); + indicesAPI.point(builder.build()); } @Override @@ -340,13 +340,13 @@ public void deactivateIndex(String indexName) throws DotDataException, IOExcepti if(LicenseUtil.getLevel() < LicenseLevel.STANDARD.level) return; - final IndiciesInfo info = indiciesAPI.loadIndicies(); - final IndiciesInfo.Builder builder = IndiciesInfo.Builder.copy(info); + final IndicesInfo info = indicesAPI.loadLegacyIndices(); + final LegacyIndicesInfo.Builder builder = LegacyIndicesInfo.Builder.copy(info); if(IndexType.SITE_SEARCH.is(indexName)) { builder.setSiteSearch(null); } - indiciesAPI.point(builder.build()); + indicesAPI.point(builder.build()); } @Override @@ -625,7 +625,7 @@ public Map getAggregations ( String indexName, String query RestHighLevelClient client = RestHighLevelClientProvider.getInstance().getClient(); if ( indexName == null ) { - indexName = indiciesAPI.loadIndicies().getSiteSearch(); + indexName = indicesAPI.loadLegacyIndices().getSiteSearch(); } if ( !indexApi.indexExists( indexName ) ) { // try using it as an alias @@ -660,7 +660,7 @@ public Map getFacets ( String indexName, String query ) thr final RestHighLevelClient client = RestHighLevelClientProvider.getInstance().getClient(); if ( indexName == null ) { - indexName = indiciesAPI.loadIndicies().getSiteSearch(); + indexName = indicesAPI.loadLegacyIndices().getSiteSearch(); } if ( !indexApi.indexExists( indexName ) ) { // try using it as an alias @@ -705,7 +705,7 @@ public void deleteOldSiteSearchIndices(){ indicesToRemove.addAll(listIndices()); //Remove Default SiteSearch Index - final IndiciesInfo info = Try.of(()->APILocator.getIndiciesAPI().loadIndicies()) + final IndicesInfo info = Try.of(()->APILocator.getIndiciesAPI().loadLegacyIndices()) .getOrNull(); if(info!=null) { diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPI.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPI.java index b796099bc5a9..c3f3676c2826 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPI.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPI.java @@ -27,9 +27,9 @@ public interface ContentletIndexAPI { */ public void checkAndInitialiazeIndex(); - public boolean createContentIndex(String indexName) throws DotIndexException, IOException; + public boolean createContentIndexLegacy(String indexName) throws DotIndexException, IOException; - public boolean createContentIndex(String indexName, int shards) throws DotIndexException, IOException; + public boolean createContentIndexLegacy(String indexName, int shards) throws DotIndexException, IOException; /** * creates new working and live indexes with reading aliases pointing to old index and write aliases @@ -130,13 +130,13 @@ public void removeContentFromIndexByContentType(final ContentType contentType) void addContentToIndex(Contentlet content, boolean deps) throws DotDataException; - BulkRequest createBulkRequest(List contentToIndex) throws DotDataException; + BulkRequest createBulkRequestLegacy(List contentToIndex) throws DotDataException; - BulkRequest createBulkRequest(); + BulkRequest createBulkRequestLegacy(); - BulkRequest appendBulkRequest(BulkRequest bulkRequest, Collection idxs) throws DotDataException; + BulkRequest appendBulkRequestLegacy(BulkRequest bulkRequest, Collection idxs) throws DotDataException; - BulkRequest appendBulkRequest(BulkRequest bulkRequest, ReindexEntry idx) throws DotDataException; + BulkRequest appendBulkRequestLegacy(BulkRequest bulkRequest, ReindexEntry idx) throws DotDataException; Optional reindexTimeElapsed(); @@ -148,7 +148,7 @@ public void removeContentFromIndexByContentType(final ContentType contentType) BulkRequest appendBulkRemoveRequest(BulkRequest bulkRequest, final ReindexEntry entry) throws DotDataException; - BulkProcessor createBulkProcessor(BulkProcessorListener bulkListener); + BulkProcessor createBulkProcessorLegacy(BulkProcessorListener bulkListener); - void appendToBulkProcessor(final BulkProcessor bulk, final Collection idxs) throws DotDataException; + void appendToBulkProcessorLegacy(final BulkProcessor bulk, final Collection idxs) throws DotDataException; } diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java index b89db47252d3..0e93da36a88d 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java @@ -12,10 +12,14 @@ import com.dotcms.api.web.HttpServletRequestThreadLocal; import com.dotcms.business.CloseDBIfOpened; import com.dotcms.business.WrapInTransaction; +import com.dotcms.cdi.CDIUtils; import com.dotcms.concurrent.DotConcurrentFactory; import com.dotcms.content.business.DotMappingException; import com.dotcms.content.elasticsearch.util.ESMappingUtilHelper; import com.dotcms.content.elasticsearch.util.RestHighLevelClientProvider; +import com.dotcms.content.opensearch.business.IndicesInfoImpl; +import com.dotcms.content.opensearch.business.IndicesInfoImpl.Builder; +import com.dotcms.content.opensearch.business.OpenSearchIndexAPI; import com.dotcms.contenttype.model.type.ContentType; import com.dotcms.exception.ExceptionUtil; import com.dotcms.rest.api.v1.DotObjectMapperProvider; @@ -116,7 +120,7 @@ public ContentletIndexAPIImpl() { } public synchronized void getRidOfOldIndex() throws DotDataException { - IndiciesInfo idxs = APILocator.getIndiciesAPI().loadIndicies(); + IndicesInfo idxs = APILocator.getIndiciesAPI().loadLegacyIndices(); if (idxs.getWorking() != null) { delete(idxs.getWorking()); } @@ -140,9 +144,8 @@ public synchronized void getRidOfOldIndex() throws DotDataException { */ @VisibleForTesting @CloseDBIfOpened - public synchronized boolean indexReady() throws DotDataException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); - + public synchronized boolean indexReadyLegacy() throws DotDataException { + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); final boolean hasWorking = Try.of(()->APILocator.getESIndexAPI().indexExists(info.getWorking())) .getOrElse(false); @@ -158,6 +161,29 @@ public synchronized boolean indexReady() throws DotDataException { return hasWorking && hasLive; } + static final class IndexStatus { + final boolean hasLive; + final boolean hasWorking; + + public IndexStatus(boolean hasLive, boolean hasWorking) { + this.hasLive = hasLive; + this.hasWorking = hasWorking; + } + } + + @CloseDBIfOpened + public synchronized boolean indexReady() throws DotDataException { + final Optional info = APILocator.getIndiciesAPI().loadIndices(); + if(info.isEmpty()){ + return false; + } + final IndicesInfo indicesInfo = info.get(); + final OpenSearchIndexAPI openSearchIndexAPI = CDIUtils.getBeanThrows(OpenSearchIndexAPI.class); + final boolean hasWorking = openSearchIndexAPI.indexExists(indicesInfo.getWorking()); + final boolean hasLive = openSearchIndexAPI.indexExists(indicesInfo.getLive()); + return hasWorking && hasLive; + } + /** * Inits the indexes and starts the reindex process if no indexes are found */ @@ -165,15 +191,14 @@ public synchronized boolean indexReady() throws DotDataException { public synchronized void checkAndInitialiazeIndex() { try { // if we don't have a working index, create it - if (!indexReady()) { + if (!indexReadyLegacy()) { Logger.info(this.getClass(), "No indexes found, creating live and working indexes"); - initIndex(); + initIndexLegacy(); } - // if there are indexes but they are empty, start reindex process if(Config.getBooleanProperty("REINDEX_IF_NO_INDEXES_FOUND", true) - && getIndexDocumentCount(APILocator.getIndiciesAPI().loadIndicies().getWorking())==0 + && getIndexDocumentCount(APILocator.getIndiciesAPI().loadLegacyIndices().getWorking())==0 ){ DotConcurrentFactory.getInstance().getSubmitter().submit(()->{ try { @@ -188,24 +213,47 @@ && getIndexDocumentCount(APILocator.getIndiciesAPI().loadIndicies().getWorking() }); } - - } catch (Exception e) { - Logger.fatal(this.getClass(), "Failed to create new indexes:" + e.getMessage(),e); + Logger.fatal(this.getClass(), "Failed to create Legacy indexes:" + e.getMessage(),e); + } + //Activate a new index + try { + final boolean ok = (IndicesAPI.isOpenSearchReadEnabled() || IndicesAPI.isOpenSearchWriteEnabled()); + if (ok) { + if (!indexReady()) { + final Builder builder = IndicesInfoImpl.builder(); + builder.withNewIndicesName(IndexType.WORKING, IndexType.LIVE); + final IndicesInfoImpl info = builder.build(); + createContentIndex(info.getWorking(), 0); + createContentIndex(info.getLive(), 0); + APILocator.getIndiciesAPI().point(info); + } + } + } catch (Exception e) { + Logger.fatal(this.getClass(), "Failed to create new indexes:" + e.getMessage(), e); } } - public synchronized boolean createContentIndex(String indexName) + public synchronized boolean createContentIndex(String indexName, int shards) + throws ElasticsearchException, IOException { + final OpenSearchIndexAPI api = CDIUtils.getBeanThrows(OpenSearchIndexAPI.class); + final org.opensearch.client.opensearch.indices.CreateIndexResponse index = api.createIndex( + indexName, shards); + System.out.println(index.index()); + return index.acknowledged(); + } + + public synchronized boolean createContentIndexLegacy(String indexName) throws ElasticsearchException, IOException { - boolean result = createContentIndex(indexName, 0); + boolean result = createContentIndexLegacy(indexName, 0); ESMappingUtilHelper.getInstance().addCustomMapping(indexName); return result; } @Override - public synchronized boolean createContentIndex(String indexName, int shards) + public synchronized boolean createContentIndexLegacy(String indexName, int shards) throws ElasticsearchException, IOException { String settings = null; @@ -241,24 +289,24 @@ public synchronized boolean createContentIndex(String indexName, int shards) * @throws ElasticsearchException if Murphy comes around * @throws DotDataException */ - private synchronized String initIndex() throws ElasticsearchException, DotDataException { - if (indexReady()) { + private synchronized String initIndexLegacy() throws ElasticsearchException, DotDataException { + if (indexReadyLegacy()) { return ""; } try { - final IndiciesInfo.Builder builder = new IndiciesInfo.Builder(); - final IndiciesInfo oldInfo = APILocator.getIndiciesAPI().loadIndicies(); + final LegacyIndicesInfo.Builder builder = new LegacyIndicesInfo.Builder(); + final IndicesInfo oldInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); if (oldInfo != null && oldInfo.getSiteSearch() != null) { builder.setSiteSearch(oldInfo.getSiteSearch()); } - final IndiciesInfo info = builder.build(); - final String timeStamp = info.createNewIndiciesName(IndexType.WORKING, IndexType.LIVE); + final LegacyIndicesInfo info = builder.build(); + final String timeStamp = info.createNewIndicesName(IndexType.WORKING, IndexType.LIVE); - createContentIndex(info.getWorking(), 0); - createContentIndex(info.getLive(), 0); + createContentIndexLegacy(info.getWorking(), 0); + createContentIndexLegacy(info.getLive(), 0); APILocator.getIndiciesAPI().point(info); @@ -337,11 +385,11 @@ public boolean reindexSwitchover(boolean forceSwitch) throws DotDataException { */ @WrapInTransaction public synchronized String fullReindexStart() throws ElasticsearchException, DotDataException { - if (indexReady() && !isInFullReindex()) { + if (indexReadyLegacy() && !isInFullReindex()) { try { - final IndiciesInfo.Builder builder = new IndiciesInfo.Builder(); - final IndiciesInfo oldInfo = APILocator.getIndiciesAPI().loadIndicies(); + final LegacyIndicesInfo.Builder builder = new LegacyIndicesInfo.Builder(); + final IndicesInfo oldInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); builder.setWorking(oldInfo.getWorking()); builder.setLive(oldInfo.getLive()); @@ -356,12 +404,12 @@ public synchronized String fullReindexStart() throws ElasticsearchException, Dot Logger.info(this, "Full reindex started by system user at " + new java.util.Date()); } - final IndiciesInfo info = builder.build(); - final String timeStamp = info.createNewIndiciesName(IndexType.REINDEX_WORKING, + final LegacyIndicesInfo info = builder.build(); + final String timeStamp = info.createNewIndicesName(IndexType.REINDEX_WORKING, IndexType.REINDEX_LIVE); - createContentIndex(info.getReindexWorking(), 0); - createContentIndex(info.getReindexLive(), 0); + createContentIndexLegacy(info.getReindexWorking(), 0); + createContentIndexLegacy(info.getReindexLive(), 0); APILocator.getIndiciesAPI().point(info); @@ -373,13 +421,13 @@ public synchronized String fullReindexStart() throws ElasticsearchException, Dot throw new ElasticsearchException(e.getMessage(), e); } } else { - return initIndex(); + return initIndexLegacy(); } } @CloseDBIfOpened public boolean isInFullReindex() throws DotDataException { - IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); return queueApi.hasReindexRecords() || (info.getReindexWorking() != null && info.getReindexLive() != null); @@ -412,7 +460,7 @@ public boolean fullReindexSwitchover(Connection conn, final boolean forceSwitch) return false; } try { - final IndiciesInfo oldInfo = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo oldInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); final String luckyServer = Try.of(() -> APILocator.getServerAPI().getOldestServer()) .getOrElse(ConfigUtils.getServerId()); if (!forceSwitch) { @@ -427,13 +475,13 @@ public boolean fullReindexSwitchover(Connection conn, final boolean forceSwitch) } } - final IndiciesInfo.Builder builder = new IndiciesInfo.Builder(); + final LegacyIndicesInfo.Builder builder = new LegacyIndicesInfo.Builder(); builder.setLive(oldInfo.getReindexLive()); builder.setWorking(oldInfo.getReindexWorking()); builder.setSiteSearch(oldInfo.getSiteSearch()); - final IndiciesInfo newInfo = builder.build(); + final LegacyIndicesInfo newInfo = builder.build(); logSwitchover(oldInfo, luckyServer); APILocator.getIndiciesAPI().point(newInfo); @@ -477,7 +525,7 @@ public boolean fullReindexSwitchover(Connection conn, final boolean forceSwitch) private long reindexTimeElapsedInLong() { try { - final IndiciesInfo oldInfo = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo oldInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); if (oldInfo.getReindexWorking() != null) { return oldInfo.getIndexTimeStamp(IndexType.REINDEX_WORKING); } @@ -505,7 +553,7 @@ public Optional reindexTimeElapsed() { return Optional.empty(); } - private void logSwitchover(final IndiciesInfo oldInfo, final String luckyServer) { + private void logSwitchover(final IndicesInfo oldInfo, final String luckyServer) { Logger.info(this, "-------------------------------"); final String myServerId = APILocator.getServerAPI().readServerId(); final Optional duration = reindexTimeElapsed(); @@ -516,10 +564,10 @@ private void logSwitchover(final IndiciesInfo oldInfo, final String luckyServer) Logger.info(this, "Switching Server Id : " + luckyServer + (luckyServer.equals(myServerId) ? " (this server) " : " (NOT this server)")); - Logger.info(this, "Old indicies : [" + esIndexApi + Logger.info(this, "Old indices : [" + esIndexApi .removeClusterIdFromName(oldInfo.getWorking()) + "," + esIndexApi .removeClusterIdFromName(oldInfo.getLive()) + "]"); - Logger.info(this, "New indicies : [" + esIndexApi + Logger.info(this, "New indices : [" + esIndexApi .removeClusterIdFromName(oldInfo.getReindexWorking()) + "," + esIndexApi .removeClusterIdFromName(oldInfo.getReindexLive()) + "]"); Logger.info(this, "-------------------------------"); @@ -620,21 +668,21 @@ public void addContentToIndex(final List contentToIndex) { } private void indexContentListNow(final List contentToIndex) { - final BulkRequest bulkRequest = createBulkRequest(contentToIndex); + final BulkRequest bulkRequest = createBulkRequestLegacy(contentToIndex); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); putToIndex(bulkRequest); CacheLocator.getESQueryCache().clearCache(); } // indexContentListNow. private void indexContentListWaitFor(final List contentToIndex) { - final BulkRequest bulkRequest = createBulkRequest(contentToIndex); + final BulkRequest bulkRequest = createBulkRequestLegacy(contentToIndex); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); putToIndex(bulkRequest); CacheLocator.getESQueryCache().clearCache(); } // indexContentListWaitFor. private void indexContentListDefer(final List contentToIndex) { - final BulkRequest bulkRequest = createBulkRequest(contentToIndex); + final BulkRequest bulkRequest = createBulkRequestLegacy(contentToIndex); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); putToIndex(bulkRequest); } // indexContentListWaitFor. @@ -679,21 +727,21 @@ public void putToIndex(final BulkRequest bulkRequest) { } @Override - public BulkRequest createBulkRequest(final List contentToIndex) { - final BulkIndexWrapper bulkIndexWrapper = new BulkIndexWrapper(createBulkRequest()); - this.appendBulkRequest(bulkIndexWrapper, contentToIndex); + public BulkRequest createBulkRequestLegacy(final List contentToIndex) { + final BulkIndexWrapper bulkIndexWrapper = new BulkIndexWrapper(createBulkRequestLegacy()); + this.appendBulkRequestLegacy(bulkIndexWrapper, contentToIndex); return bulkIndexWrapper.getRequestBuilder(); } @Override - public BulkRequest createBulkRequest() { + public BulkRequest createBulkRequestLegacy() { final BulkRequest bulkRequest = new BulkRequest(); bulkRequest.setRefreshPolicy(RefreshPolicy.NONE); return bulkRequest; } - public BulkProcessor createBulkProcessor(final BulkProcessorListener bulkProcessorListener) { + public BulkProcessor createBulkProcessorLegacy(final BulkProcessorListener bulkProcessorListener) { BulkProcessor.Builder builder = BulkProcessor.builder( (request, bulkListener) -> RestHighLevelClientProvider.getInstance().getClient() @@ -717,17 +765,17 @@ public BulkProcessor createBulkProcessor(final BulkProcessorListener bulkProcess } @Override - public BulkRequest appendBulkRequest(final BulkRequest bulkRequest, + public BulkRequest appendBulkRequestLegacy(final BulkRequest bulkRequest, final Collection idxs) throws DotDataException { for (ReindexEntry idx : idxs) { - appendBulkRequest(bulkRequest, idx); + appendBulkRequestLegacy(bulkRequest, idx); } return bulkRequest; } - public void appendToBulkProcessor(final BulkProcessor bulk, final Collection idxs) + public void appendToBulkProcessorLegacy(final BulkProcessor bulk, final Collection idxs) throws DotDataException { for (ReindexEntry idx : idxs) { @@ -736,16 +784,16 @@ public void appendToBulkProcessor(final BulkProcessor bulk, final Collection versions = APILocator.getVersionableAPI() .findContentletVersionInfos(idx.getIdentToIndex()); @@ -789,7 +837,7 @@ public void appendBulkRequest(final BulkIndexWrapper bulk, final ReindexEntry id idx .getPriority())); contentlet.setIndexPolicy(IndexPolicy.DEFER); - addBulkRequest(bulk, List.of(contentlet), idx.isReindex()); + addBulkRequestLegacy(bulk, List.of(contentlet), idx.isReindex()); } } catch (final Exception e) { // An error occurred when trying to reindex the Contentlet. Flag it as "failed" @@ -805,18 +853,18 @@ public BulkProcessor appendToBulkProcessor(BulkProcessor bulk, final ReindexEntr if (idx.isDelete()) { appendBulkRemoveRequest(bulkIndexWrapper, idx); } else { - appendBulkRequest(bulkIndexWrapper, idx); + appendBulkRequestLegacy(bulkIndexWrapper, idx); } return bulkIndexWrapper.getBulkProcessor(); } - private void appendBulkRequest(final BulkIndexWrapper bulk, + private void appendBulkRequestLegacy(final BulkIndexWrapper bulk, final List contentToIndex) { - this.addBulkRequest(bulk, contentToIndex, false); + this.addBulkRequestLegacy(bulk, contentToIndex, false); } - private void addBulkRequest(final BulkIndexWrapper bulk, final List contentToIndex, + private void addBulkRequestLegacy(final BulkIndexWrapper bulk, final List contentToIndex, final boolean forReindex) { if (contentToIndex != null && !contentToIndex.isEmpty()) { Logger.debug(this.getClass(), @@ -843,8 +891,8 @@ private void addBulkRequest(final BulkIndexWrapper bulk, final List .getCurrentStackTraceAsString(Config.getIntProperty("stacktracelimit", 10)) + "\n"); - final IndiciesInfo info = Sneaky - .sneak(() -> APILocator.getIndiciesAPI().loadIndicies()); + final IndicesInfo info = Sneaky + .sneak(() -> APILocator.getIndiciesAPI().loadLegacyIndices()); String mapping = null; try { @@ -970,7 +1018,7 @@ public void appendBulkRemoveRequest(final BulkIndexWrapper bulk, final ReindexEn throws DotDataException { final List languages = APILocator.getLanguageAPI().getLanguages(); final List variants = APILocator.getVariantAPI().getVariants(); - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); // delete for every language and in every index for (Language language : languages) { @@ -1075,7 +1123,7 @@ private void removeContentAndProcessDependencies(final Contentlet contentlet, final String id = builder(contentlet.getIdentifier(), StringPool.UNDERLINE, contentlet.getLanguageId(), StringPool.UNDERLINE, contentlet.getVariantId()) .toString(); - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); final BulkRequest bulkRequest = new BulkRequest(); // we want to wait until the content is already indexed @@ -1220,7 +1268,7 @@ public void removeContentFromIndexByContentType(final ContentType contentType) throws DotDataException { final String structureName = contentType.variable(); - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); // collecting indexes final List idxs = new ArrayList<>(); @@ -1256,19 +1304,19 @@ public void fullReindexAbort() { return; } - IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); - final IndiciesInfo.Builder builder = new IndiciesInfo.Builder(); + final LegacyIndicesInfo.Builder builder = new LegacyIndicesInfo.Builder(); builder.setWorking(info.getWorking()); builder.setLive(info.getLive()); builder.setSiteSearch(info.getSiteSearch()); - IndiciesInfo newinfo = builder.build(); + LegacyIndicesInfo newInfo = builder.build(); info.getReindexWorking(); info.getReindexLive(); - APILocator.getIndiciesAPI().point(newinfo); + APILocator.getIndiciesAPI().point(newInfo); } catch (Exception e) { throw new ElasticsearchException(e.getMessage(), e); } @@ -1295,8 +1343,8 @@ public List listDotCMSIndices() { public void activateIndex(final String indexName) throws DotDataException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); - final IndiciesInfo.Builder builder = IndiciesInfo.Builder.copy(info); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); + final LegacyIndicesInfo.Builder builder = LegacyIndicesInfo.Builder.copy(info); if (indexName == null) { throw new DotRuntimeException("Index cannot be null"); } @@ -1324,8 +1372,8 @@ public void activateIndex(final String indexName) throws DotDataException { } public void deactivateIndex(String indexName) throws DotDataException, IOException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); - final IndiciesInfo.Builder builder = IndiciesInfo.Builder.copy(info); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); + final LegacyIndicesInfo.Builder builder = LegacyIndicesInfo.Builder.copy(info); if (IndexType.WORKING.is(indexName)) { builder.setWorking(null); @@ -1356,7 +1404,7 @@ public long getIndexDocumentCount(final String indexName) { public synchronized List getCurrentIndex() throws DotDataException { final List newIdx = new ArrayList<>(); - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); newIdx.add(esIndexApi.removeClusterIdFromName(info.getWorking())); newIdx.add(esIndexApi.removeClusterIdFromName(info.getLive())); return newIdx; @@ -1364,7 +1412,7 @@ public synchronized List getCurrentIndex() throws DotDataException { public synchronized List getNewIndex() throws DotDataException { final List newIdx = new ArrayList<>(); - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); if (info.getReindexWorking() != null) { newIdx.add(esIndexApi.removeClusterIdFromName(info.getReindexWorking())); @@ -1376,7 +1424,7 @@ public synchronized List getNewIndex() throws DotDataException { } public String getActiveIndexName(final String type) throws DotDataException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); if (IndexType.WORKING.is(type)) { return esIndexApi.removeClusterIdFromName(info.getWorking()); @@ -1387,4 +1435,6 @@ public String getActiveIndexName(final String type) throws DotDataException { return null; } + + } diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentFactoryImpl.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentFactoryImpl.java index 1b59ccde5e4e..2cf09b692a49 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentFactoryImpl.java @@ -950,7 +950,7 @@ protected List findAllCurrent (final int offset, final int limit ) t final String indexToHit; try { - indexToHit = APILocator.getIndiciesAPI().loadIndicies().getWorking(); + indexToHit = APILocator.getIndiciesAPI().loadLegacyIndices().getWorking(); } catch(DotDataException ee) { Logger.fatal(this, "Can't get indicies information",ee); @@ -1696,9 +1696,9 @@ private CountRequest getCountRequest(final String queryString) { private String inferIndexToHit(final String query) { // we check the query to figure out which indexes to hit - final IndiciesInfo info; + final IndicesInfo info; try { - info = APILocator.getIndiciesAPI().loadIndicies(); + info = APILocator.getIndiciesAPI().loadLegacyIndices(); } catch (DotDataException e) { throw new DotRuntimeException(e); } diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESIndexAPI.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESIndexAPI.java index cb0374508a6e..7e7b90e1a649 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESIndexAPI.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESIndexAPI.java @@ -1,7 +1,7 @@ package com.dotcms.content.elasticsearch.business; import static com.dotcms.content.elasticsearch.business.ESIndexHelper.SNAPSHOT_PREFIX; -import static com.dotcms.content.elasticsearch.business.IndiciesInfo.CLUSTER_PREFIX; +import static com.dotcms.content.elasticsearch.business.LegacyIndicesInfo.CLUSTER_PREFIX; import static com.dotcms.util.DotPreconditions.checkArgument; import com.dotcms.cluster.ClusterUtils; @@ -26,12 +26,10 @@ import com.rainerhahnekamp.sneakythrow.Sneaky; import io.vavr.Lazy; import io.vavr.control.Try; -import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.net.URL; import java.nio.file.Files; @@ -47,15 +45,11 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.ZipException; import java.util.zip.ZipFile; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.util.Strings; import org.apache.tools.zip.ZipEntry; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; @@ -84,8 +78,6 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; @@ -458,7 +450,7 @@ private void deleteLiveWorkinSetFromList(List indicesToRemove, String in } private void removeActiveLiveAndWorkingFromList(List indices) { - final IndiciesInfo info = Try.of(()->APILocator.getIndiciesAPI().loadIndicies()) + final IndicesInfo info = Try.of(()->APILocator.getIndiciesAPI().loadLegacyIndices()) .getOrNull(); if(info!=null) { @@ -909,7 +901,7 @@ boolean hasClusterPrefix(final String indexName) { /** * Given an alias or index name that might contain a cluster id prefix - * (format: {@link IndiciesInfo#CLUSTER_PREFIX CLUSTER_PREFIX}_{id}.{name}), + * (format: {@link LegacyIndicesInfo#CLUSTER_PREFIX CLUSTER_PREFIX}_{id}.{name}), * this method will return the name without the prefix. In case of name is null, an empty string * will be returned * @param name Index name or alias with the cluster id prefix @@ -1008,7 +1000,7 @@ public File createSnapshot(String repositoryName, String snapshotName, String in /** * Given an alias or index name, this method will return the full name including the cluster id, - * using this format: {@link IndiciesInfo#CLUSTER_PREFIX CLUSTER_PREFIX}_{id}.{name} + * using this format: {@link LegacyIndicesInfo#CLUSTER_PREFIX CLUSTER_PREFIX}_{id}.{name} * @param name Index name or alias * @return Index name or alias with the cluster id prefix */ diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndexType.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndexType.java index 74c6d77dc8ea..b7d48917e44f 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndexType.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndexType.java @@ -1,6 +1,6 @@ package com.dotcms.content.elasticsearch.business; -import static com.dotcms.content.elasticsearch.business.IndiciesInfo.CLUSTER_PREFIX; +import static com.dotcms.content.elasticsearch.business.LegacyIndicesInfo.CLUSTER_PREFIX; import com.dotcms.enterprise.cluster.ClusterFactory; diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesAPI.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesAPI.java new file mode 100644 index 000000000000..f5573414bc2a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesAPI.java @@ -0,0 +1,71 @@ +package com.dotcms.content.elasticsearch.business; + +import static com.dotcms.featureflag.FeatureFlagName.FEATURE_FLAG_OPEN_SEARCH_READ; +import static com.dotcms.featureflag.FeatureFlagName.FEATURE_FLAG_OPEN_SEARCH_WRITE; + +import com.dotmarketing.util.Config; +import java.sql.Connection; + + +import com.dotmarketing.exception.DotDataException; +import java.util.Optional; + +/** + * An API to store and retrieve information about current Elastic Search Indices + * + * @author Jorge Urdaneta + */ +public interface IndicesAPI { + + /** + * Loads information about legacy indices. + * + * @return an {@code IndicesInfo} object containing details about the legacy indices + * @throws DotDataException if there is an error while fetching the indices information + */ + IndicesInfo loadLegacyIndices() throws DotDataException; + + /** + * Loads information about legacy indices from the database connection provided. + * + * @param conn the database connection used to fetch the legacy indices information + * @return an {@code IndicesInfo} object containing details about the legacy indices + * @throws DotDataException if there is an error while fetching the indices information + */ + IndicesInfo loadLegacyIndices(Connection conn) throws DotDataException; + + /** + * Loads the new indices information. + * @return + * @throws DotDataException + */ + Optional loadIndices() throws DotDataException; + + /** + * Updates the current indices information with the provided {@code IndicesInfo} object. + * This method can be used to reassign or redefine indices details like live, working, + * reindexing, or site search indices. + * + * @param newInfo the new indices information to set, encapsulated in an {@code IndicesInfo} object + * @throws DotDataException if there is an error updating the indices information + */ + void point(IndicesInfo newInfo) throws DotDataException; + + + /** + * + * @return + */ + static boolean isOpenSearchWriteEnabled(){ + return Config.getBooleanProperty(FEATURE_FLAG_OPEN_SEARCH_WRITE, false); + } + + /** + * + * @return + */ + static boolean isOpenSearchReadEnabled(){ + return Config.getBooleanProperty(FEATURE_FLAG_OPEN_SEARCH_READ, false); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesAPIImpl.java similarity index 50% rename from dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesAPIImpl.java rename to dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesAPIImpl.java index 7054534c96cd..2147427cb481 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesAPIImpl.java @@ -11,28 +11,34 @@ import com.liferay.portal.model.User; import com.liferay.portal.util.PortalUtil; import io.vavr.control.Try; +import java.util.Optional; -public class IndiciesAPIImpl implements IndiciesAPI { +public class IndicesAPIImpl implements IndicesAPI { - protected final IndiciesFactory ifac; + protected final IndicesFactory indicesFactory; - public IndiciesAPIImpl(){ - ifac = FactoryLocator.getIndiciesFactory(); + public IndicesAPIImpl(){ + indicesFactory = FactoryLocator.getIndiciesFactory(); } @CloseDBIfOpened - public IndiciesInfo loadIndicies() throws DotDataException { - return loadIndicies(null); + public LegacyIndicesInfo loadLegacyIndices() throws DotDataException { + return loadLegacyIndices(null); } @CloseDBIfOpened - public IndiciesInfo loadIndicies(Connection conn) throws DotDataException { - return ifac.loadIndicies(conn); + public LegacyIndicesInfo loadLegacyIndices(Connection conn) throws DotDataException { + return indicesFactory.loadLegacyIndices(conn); + } + + @CloseDBIfOpened + public Optional loadIndices() throws DotDataException{ + return indicesFactory.loadIndices(); } @WrapInTransaction - public synchronized void point(IndiciesInfo newInfo) throws DotDataException { + public synchronized void point(IndicesInfo newInfo) throws DotDataException { final User currentUser = Try.of(() -> PortalUtil.getUser(HttpServletRequestThreadLocal.INSTANCE.getRequest())) .getOrNull(); @@ -46,10 +52,17 @@ public synchronized void point(IndiciesInfo newInfo) throws DotDataException { newInfo.getReindexWorking(), newInfo.getReindexLive(), newInfo.getSiteSearch()); - - Logger.info(this, "Indices configuration updated by user: " + userInfo + " - " + indexInfo + " at " + new java.util.Date()); - - ifac.point(newInfo); + + if(newInfo.isLegacy()) { + Logger.info(this, + "Legacy Indices configuration updated by user: " + userInfo + " - " + indexInfo + + " at " + new java.util.Date()); + } else { + Logger.info(this, + "New Indices configuration updated by user: " + userInfo + " - " + indexInfo + + " at " + new java.util.Date()); + } + indicesFactory.point(newInfo); } } diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesCache.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesCache.java new file mode 100644 index 000000000000..1b83d8dd8e83 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesCache.java @@ -0,0 +1,8 @@ +package com.dotcms.content.elasticsearch.business; + +import com.dotmarketing.business.Cachable; + +public interface IndicesCache extends Cachable { + LegacyIndicesInfo get(); + void put(LegacyIndicesInfo info); +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesCacheImpl.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesCacheImpl.java similarity index 70% rename from dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesCacheImpl.java rename to dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesCacheImpl.java index af0fcd5055b5..7bfaba8a5729 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesCacheImpl.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesCacheImpl.java @@ -4,14 +4,14 @@ import com.dotmarketing.business.DotCacheAdministrator; import com.dotmarketing.util.Logger; -public class IndiciesCacheImpl implements IndiciesCache { +public class IndicesCacheImpl implements IndicesCache { protected final DotCacheAdministrator cache; - protected final String primaryGroup = "IndiciesCache"; + protected final String primaryGroup = "IndicesCache"; protected final String[] groupNames = {primaryGroup}; - public IndiciesCacheImpl() { + public IndicesCacheImpl() { cache = CacheLocator.getCacheAdministrator(); } @@ -27,10 +27,10 @@ public void clearCache() { cache.flushGroup(primaryGroup); } - public IndiciesInfo get() { - IndiciesInfo info=null; + public LegacyIndicesInfo get() { + LegacyIndicesInfo info=null; try { - info=(IndiciesInfo)cache.get(primaryGroup+"info", primaryGroup); + info=(LegacyIndicesInfo)cache.get(primaryGroup+"info", primaryGroup); } catch(Exception ex) { Logger.warn(this, "can't get cache entry",ex); @@ -38,7 +38,7 @@ public IndiciesInfo get() { return info; } - public void put(IndiciesInfo info) { + public void put(LegacyIndicesInfo info) { cache.put(primaryGroup+"info", info, primaryGroup); } } diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesFactory.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesFactory.java similarity index 59% rename from dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesFactory.java rename to dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesFactory.java index 8852a12466e1..8e0b06e3a0e1 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesFactory.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesFactory.java @@ -1,12 +1,12 @@ package com.dotcms.content.elasticsearch.business; +import com.dotcms.content.opensearch.business.IndicesInfoImpl; import java.sql.Connection; import java.sql.SQLException; import java.util.List; import java.util.Map; import com.dotcms.business.CloseDBIfOpened; -import com.dotmarketing.business.APILocator; import com.dotmarketing.business.CacheLocator; import com.dotmarketing.common.db.DotConnect; import com.dotmarketing.db.DbConnectionFactory; @@ -15,32 +15,33 @@ import com.dotmarketing.util.Logger; import io.vavr.control.Try; +import java.util.Optional; import org.apache.commons.beanutils.PropertyUtils; -public class IndiciesFactory { +public class IndicesFactory { - protected static IndiciesCache cache = CacheLocator.getIndiciesCache(); + protected static IndicesCache cache = CacheLocator.getIndiciesCache(); - public IndiciesInfo loadIndicies() throws DotDataException { - return loadIndicies(null); + public LegacyIndicesInfo loadLegacyIndices() throws DotDataException { + return loadLegacyIndices(null); } @CloseDBIfOpened - public IndiciesInfo loadIndicies(Connection conn) throws DotDataException { - IndiciesInfo info = cache.get(); + public LegacyIndicesInfo loadLegacyIndices(Connection conn) throws DotDataException { + LegacyIndicesInfo info = cache.get(); if (info == null) { // build it once - synchronized (this.getClass()) { + synchronized (IndicesFactory.class) { if (conn == null) { conn = DbConnectionFactory.getConnection(); } info = cache.get(); if (info == null) { - final IndiciesInfo.Builder builder = new IndiciesInfo.Builder(); + final LegacyIndicesInfo.Builder builder = new LegacyIndicesInfo.Builder(); final DotConnect dc = new DotConnect(); - dc.setSQL("SELECT index_name,index_type FROM indicies"); + dc.setSQL("SELECT index_name,index_type FROM indicies WHERE index_version is NULL"); final List> results = dc.loadResults(conn); for (Map rr : results) { String name = (String) rr.get("index_name"); @@ -66,21 +67,51 @@ public IndiciesInfo loadIndicies(Connection conn) throws DotDataException { return info; } + @CloseDBIfOpened + public Optional loadIndices() throws DotDataException { + IndicesInfoImpl info = null; + // build it once + synchronized (IndicesFactory.class) { + final IndicesInfoImpl.Builder builder = new IndicesInfoImpl.Builder(); + final DotConnect dc = new DotConnect(); + dc.setSQL("SELECT index_name,index_type FROM indicies WHERE index_version = '" + + IndicesInfo.OPEN_SEARCH_VERSION + "' "); + @SuppressWarnings("unchecked") final List> results = dc.loadResults(); + if(results.isEmpty()){ + return Optional.empty(); + } + for (Map rr : results) { + String name = (String) rr.get("index_name"); + String type = (String) rr.get("index_type"); + if (type.equalsIgnoreCase(IndexType.WORKING.toString())) { + builder.working(name); + } else if (type.equalsIgnoreCase(IndexType.LIVE.toString())) { + builder.live(name); + } else if (type.equalsIgnoreCase(IndexType.REINDEX_LIVE.toString())) { + builder.reindexLive(name); + } else if (type.equalsIgnoreCase(IndexType.REINDEX_WORKING.toString())) { + builder.reindexWorking(name); + } else if (type.equalsIgnoreCase(IndexType.SITE_SEARCH.toString())) { + builder.siteSearch(name); + } + } + info = builder.build(); + } + return Optional.of(info); + } - public void point(final IndiciesInfo newInfo) throws DotDataException { + public void point(final IndicesInfo newInfo) throws DotDataException { Connection conn = null; try { conn = DbConnectionFactory.getDataSource().getConnection(); conn.setAutoCommit(false); - if (DbConnectionFactory.isMySql()) { - conn.setTransactionIsolation(conn.TRANSACTION_READ_COMMITTED); - } - if(newInfo==null || newInfo.equals(loadIndicies(conn))) { + + if(newInfo==null || newInfo.equals(loadLegacyIndices(conn))) { return; } DotConnect dc = new DotConnect(); - final String insertSQL = "INSERT INTO indicies VALUES(?,?)"; + final String insertSQL = "INSERT INTO indicies VALUES(?,?,?)"; final String deleteSQL = "DELETE from indicies where index_type=? or index_name=?"; for (IndexType type : IndexType.values()) { final String indexType = type.toString().toLowerCase(); @@ -89,7 +120,7 @@ public void point(final IndiciesInfo newInfo) throws DotDataException { dc.setSQL(deleteSQL).addParam(indexType).addParam(newValue).loadResult(conn); if (newValue != null) { - dc.setSQL(insertSQL).addParam(newValue).addParam(indexType).loadResult(conn); + dc.setSQL(insertSQL).addParam(newValue).addParam(indexType).addParam(newInfo.version()).loadResult(conn); } } diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesInfo.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesInfo.java new file mode 100644 index 000000000000..55f57ddfbf46 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndicesInfo.java @@ -0,0 +1,26 @@ +package com.dotcms.content.elasticsearch.business; + +import java.util.Map; + +public interface IndicesInfo { + + String OPEN_SEARCH_VERSION = "3.x"; + + String getLive(); + + String getWorking(); + + String getReindexLive(); + + String getReindexWorking(); + + String getSiteSearch(); + + String version(); + + default boolean isLegacy(){ return !OPEN_SEARCH_VERSION.equals(version()); } + + long getIndexTimeStamp(IndexType indexType); + + Map asMap(); +} diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesAPI.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesAPI.java deleted file mode 100644 index 3a11e171d11c..000000000000 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesAPI.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.dotcms.content.elasticsearch.business; - -import java.io.Serializable; -import java.sql.Connection; -import java.util.HashMap; -import java.util.Map; - - -import com.dotmarketing.exception.DotDataException; - -import io.vavr.control.Try; - -/** - * An API to store and retrieve information about current Elastic Search Indicies - * - * @author Jorge Urdaneta - */ -public interface IndiciesAPI { - - /** - * Returns IndiciesInfo instance with index names stored. - * - * @return IndiciesInfo instance - */ - public IndiciesInfo loadIndicies() throws DotDataException; - - public IndiciesInfo loadIndicies(Connection conn) throws DotDataException; - - /** - * Updates the información about ES indicies. - * - * @param info - */ - public void point(IndiciesInfo newInfo) throws DotDataException; - -} diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesCache.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesCache.java deleted file mode 100644 index 7687408f7d68..000000000000 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesCache.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.dotcms.content.elasticsearch.business; - -import com.dotmarketing.business.Cachable; - -public interface IndiciesCache extends Cachable { - public IndiciesInfo get(); - public void put(IndiciesInfo info); -} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesInfo.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/LegacyIndicesInfo.java similarity index 77% rename from dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesInfo.java rename to dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/LegacyIndicesInfo.java index ce8b55c3eeee..545da0f32c9d 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesInfo.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/LegacyIndicesInfo.java @@ -24,10 +24,10 @@ * - INDEX_TYPE_PREFIX: Prefix define for the Index's typs, also see {@link IndexType} * - TIME_STAMP: current time stamop when the index was created */ -public class IndiciesInfo implements Serializable { +public class LegacyIndicesInfo implements IndicesInfo, Serializable { /** - * Build a new {@link IndiciesInfo} instance + * Build a new {@link LegacyIndicesInfo} instance */ public static class Builder { private String live, working, reindexLive, reindexWorking, siteSearch; @@ -57,12 +57,12 @@ public Builder setSiteSearch(String siteSearch) { return this; } - public IndiciesInfo build() { - return new IndiciesInfo(this); + public LegacyIndicesInfo build() { + return new LegacyIndicesInfo(this); } - public static Builder copy(final IndiciesInfo info) { - final IndiciesInfo.Builder builder = new IndiciesInfo.Builder(); + public static Builder copy(final IndicesInfo info) { + final LegacyIndicesInfo.Builder builder = new LegacyIndicesInfo.Builder(); builder.setWorking(info.getWorking()); builder.setLive(info.getLive()); builder.setReindexWorking(info.getReindexWorking()); @@ -74,16 +74,16 @@ public static Builder copy(final IndiciesInfo info) { public static final SimpleDateFormat timestampFormatter = new SimpleDateFormat("yyyyMMddHHmmss"); public static final String CLUSTER_PREFIX = "cluster_"; - private final static String INDEX_NAME_PATTERN = CLUSTER_PREFIX + "%s.%s_%s"; + private static final String INDEX_NAME_PATTERN = CLUSTER_PREFIX + "%s.%s_%s"; - private final Map indiciesNames = new HashMap<>(); + private final Map indicesNames = new HashMap<>(); - public IndiciesInfo(final Builder builder) { - this.indiciesNames.put(IndexType.LIVE, builder.live); - this.indiciesNames.put(IndexType.WORKING, builder.working); - this.indiciesNames.put(IndexType.REINDEX_LIVE, builder.reindexLive); - this.indiciesNames.put(IndexType.REINDEX_WORKING, builder.reindexWorking); - this.indiciesNames.put(IndexType.SITE_SEARCH, builder.siteSearch); + public LegacyIndicesInfo(final Builder builder) { + this.indicesNames.put(IndexType.LIVE, builder.live); + this.indicesNames.put(IndexType.WORKING, builder.working); + this.indicesNames.put(IndexType.REINDEX_LIVE, builder.reindexLive); + this.indicesNames.put(IndexType.REINDEX_WORKING, builder.reindexWorking); + this.indicesNames.put(IndexType.SITE_SEARCH, builder.siteSearch); } /** @@ -91,7 +91,7 @@ public IndiciesInfo(final Builder builder) { * @return */ public String getLive() { - return this.indiciesNames.get(IndexType.LIVE); + return this.indicesNames.get(IndexType.LIVE); } /** @@ -99,7 +99,7 @@ public String getLive() { * @return */ public String getWorking() { - return this.indiciesNames.get(IndexType.WORKING); + return this.indicesNames.get(IndexType.WORKING); } /** @@ -107,7 +107,7 @@ public String getWorking() { * @return */ public String getReindexLive() { - return this.indiciesNames.get(IndexType.REINDEX_LIVE); + return this.indicesNames.get(IndexType.REINDEX_LIVE); } /** @@ -115,7 +115,7 @@ public String getReindexLive() { * @return */ public String getReindexWorking() { - return this.indiciesNames.get(IndexType.REINDEX_WORKING); + return this.indicesNames.get(IndexType.REINDEX_WORKING); } /** @@ -123,7 +123,16 @@ public String getReindexWorking() { * @return */ public String getSiteSearch() { - return this.indiciesNames.get(IndexType.SITE_SEARCH); + return this.indicesNames.get(IndexType.SITE_SEARCH); + } + + /** + * Legacy does not have an implicit version, so null is assumed as legacy + * @return + */ + @Override + public String version() { + return null; } /** @@ -135,7 +144,7 @@ public String getSiteSearch() { public long getIndexTimeStamp(final IndexType indexType) { Date startTime; try { - final String indexName = indiciesNames.get(indexType); + final String indexName = indicesNames.get(indexType); final String indexTimestamp = indexName.substring(indexName.lastIndexOf("_") + 1); startTime = timestampFormatter.parse(indexTimestamp); @@ -150,10 +159,10 @@ public long getIndexTimeStamp(final IndexType indexType) { * * @param indiciesType types for what you want to create a new name * @return timestamp for the newly indicies, to get the newly indices name use one of the follow methods: - * {@link IndiciesInfo#getLive()}, {@link IndiciesInfo#getWorking()}, {@link IndiciesInfo#getReindexLive()}, - * {@link IndiciesInfo#getReindexWorking()} {@link IndiciesInfo#getSiteSearch()} + * {@link LegacyIndicesInfo#getLive()}, {@link LegacyIndicesInfo#getWorking()}, {@link LegacyIndicesInfo#getReindexLive()}, + * {@link LegacyIndicesInfo#getReindexWorking()} {@link LegacyIndicesInfo#getSiteSearch()} */ - public String createNewIndiciesName(final IndexType... indiciesType) { + public String createNewIndicesName(final IndexType... indiciesType) { final String timeStamp = timestampFormatter.format(new Date()); for (final IndexType indexType : indiciesType) { @@ -163,7 +172,7 @@ public String createNewIndiciesName(final IndexType... indiciesType) { indexType.getPrefix(), timeStamp); - this.indiciesNames.put(indexType, indexName); + this.indicesNames.put(indexType, indexName); } return timeStamp; @@ -202,7 +211,7 @@ public boolean equals(Object obj) { return false; if (getClass() != obj.getClass()) return false; - IndiciesInfo other = (IndiciesInfo) obj; + LegacyIndicesInfo other = (LegacyIndicesInfo) obj; if (this.getLive() == null) { if (other.getLive() != null) return false; diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/util/ESReindexationProcessStatus.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/util/ESReindexationProcessStatus.java index b8380d67e2cb..60a14cd3f58e 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/util/ESReindexationProcessStatus.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/util/ESReindexationProcessStatus.java @@ -1,27 +1,25 @@ package com.dotcms.content.elasticsearch.util; -import com.dotcms.content.elasticsearch.business.ESIndexAPI; -import java.io.Serializable; -import java.util.Hashtable; -import java.util.Map; - import com.dotcms.business.CloseDBIfOpened; import com.dotcms.content.elasticsearch.business.ContentletIndexAPIImpl; -import com.dotcms.content.elasticsearch.business.IndiciesInfo; +import com.dotcms.content.elasticsearch.business.ESIndexAPI; +import com.dotcms.content.elasticsearch.business.IndicesInfo; import com.dotmarketing.business.APILocator; import com.dotmarketing.common.db.DotConnect; import com.dotmarketing.exception.DotDataException; -import io.vavr.control.Try; +import java.io.Serializable; +import java.util.Hashtable; +import java.util.Map; public class ESReindexationProcessStatus implements Serializable { private static final ContentletIndexAPIImpl indexAPI = new ContentletIndexAPIImpl(); - public synchronized static boolean inFullReindexation() throws DotDataException { + public static synchronized boolean inFullReindexation() throws DotDataException { return indexAPI.isInFullReindex(); } @CloseDBIfOpened - public synchronized static int getContentCountToIndex() throws DotDataException { + public static synchronized int getContentCountToIndex() throws DotDataException { DotConnect dc = new DotConnect(); dc.setSQL("select count(*) as cc from contentlet_version_info"); @@ -43,7 +41,7 @@ public static int getLastIndexationProgress(int countToIndex) throws DotDataExce @CloseDBIfOpened public static String currentIndexPath() throws DotDataException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); final ESIndexAPI esIndexAPI = APILocator.getESIndexAPI(); return "[" + esIndexAPI.removeClusterIdFromName(info.getWorking()) + "," + esIndexAPI .removeClusterIdFromName(info.getLive()) + "]"; @@ -51,7 +49,7 @@ public static String currentIndexPath() throws DotDataException { @CloseDBIfOpened public static String getNewIndexPath() throws DotDataException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); final ESIndexAPI esIndexAPI = APILocator.getESIndexAPI(); return "[" + esIndexAPI.removeClusterIdFromName(info.getReindexWorking()) + "," + esIndexAPI.removeClusterIdFromName(info.getReindexLive()) + "]"; diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/business/IndexStats.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/IndexStats.java new file mode 100644 index 000000000000..05f7548d745d --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/IndexStats.java @@ -0,0 +1,63 @@ +package com.dotcms.content.opensearch.business; + +/** + * Represents statistics for an OpenSearch index + * + * @author fabrizio + */ +public class IndexStats { + + private final String indexName; + private final long documentCount; + private final long size; + private final String prettySize; + + public IndexStats(final String indexName, final long documentCount, final long size) { + this.indexName = indexName; + this.documentCount = documentCount; + this.size = size; + this.prettySize = formatBytes(size); + } + + public long getDocumentCount() { + return documentCount; + } + + public String getSize() { + return prettySize; + } + + public long getSizeRaw() { + return size; + } + + public String getIndexName() { + return indexName; + } + + /** + * Format bytes to human readable string + */ + private String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + "b"; + } + + String[] units = {"b", "kb", "mb", "gb", "tb", "pb"}; + double size = bytes; + int unitIndex = 0; + + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex++; + } + + if (size >= 100) { + return String.format("%.0f%s", size, units[unitIndex]); + } else if (size >= 10) { + return String.format("%.1f%s", size, units[unitIndex]); + } else { + return String.format("%.2f%s", size, units[unitIndex]); + } + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/business/IndicesInfoImpl.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/IndicesInfoImpl.java new file mode 100644 index 000000000000..7a3143fde06a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/IndicesInfoImpl.java @@ -0,0 +1,351 @@ +package com.dotcms.content.opensearch.business; + +import com.dotcms.content.elasticsearch.business.IndexType; +import com.dotcms.content.elasticsearch.business.IndicesInfo; +import com.dotcms.enterprise.cluster.ClusterFactory; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.UtilMethods; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.util.Date; +import java.util.EnumMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Modern immutable implementation for tracking search indices with versioning support. + * + * This class provides an immutable, thread-safe way to manage search index information + * with enhanced features including: + * - Version tracking for index evolution + * - Modern Java time API support + * - Immutable data structures + * - Builder pattern for flexible construction + * + * Index naming convention: cluster_.__v + * + * @author fabrizio + */ +public final class IndicesInfoImpl implements IndicesInfo { + + // Modern timestamp format for better readability and sortability + private static final String CLUSTER_PREFIX = "cluster_"; + private static final String INDEX_NAME_PATTERN = CLUSTER_PREFIX + "%s.%s_%s_v%s"; + private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); + + // Immutable fields + private final String live; + private final String working; + private final String reindexLive; + private final String reindexWorking; + private final String siteSearch; + private final String version; + + /** + * Private constructor - use Builder + */ + private IndicesInfoImpl(Builder builder) { + this.live = builder.live; + this.working = builder.working; + this.reindexLive = builder.reindexLive; + this.reindexWorking = builder.reindexWorking; + this.siteSearch = builder.siteSearch; + this.version = builder.version; + } + + /** + * Create a builder for ModernIndicesInfo + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Create a copy builder from existing instance + */ + public Builder toBuilder() { + return new Builder() + .live(this.live) + .working(this.working) + .reindexLive(this.reindexLive) + .reindexWorking(this.reindexWorking) + .siteSearch(this.siteSearch) + .version(this.version); + } + + // Getters + + public String live() { + return live; + } + + public String working() { + return working; + } + + public String reindexLive() { + return reindexLive; + } + + public String reindexWorking() { + return reindexWorking; + } + + public String siteSearch() { + return siteSearch; + } + + public String version() { + return version; + } + + // IndicesInfo interface implementation + + @Override + public String getLive() { + return live; + } + + @Override + public String getWorking() { + return working; + } + + @Override + public String getReindexLive() { + return reindexLive; + } + + @Override + public String getReindexWorking() { + return reindexWorking; + } + + @Override + public String getSiteSearch() { + return siteSearch; + } + + @Override + public long getIndexTimeStamp(IndexType indexType) { + Optional indexName = getIndexNameForType(indexType); + if (indexName.isEmpty()) { + throw new DotRuntimeException("No index found for type: " + indexType); + } + + try { + String timestamp = extractTimestampFromIndexName(indexName.get()); + Date startTime = simpleDateFormat.parse(timestamp); + return System.currentTimeMillis() - startTime.getTime(); + } catch (ParseException e) { + Logger.error(IndicesInfoImpl.class, "Error parsing timestamp from index name: " + indexName, e); + throw new DotRuntimeException("Failed to parse timestamp from index: " + indexName, e); + } + } + + @Override + public Map asMap() { + Map result = new EnumMap<>(IndexType.class); + + if (UtilMethods.isSet(live)) { + result.put(IndexType.LIVE, live); + } + if (UtilMethods.isSet(working)) { + result.put(IndexType.WORKING, working); + } + if (UtilMethods.isSet(reindexLive)) { + result.put(IndexType.REINDEX_LIVE, reindexLive); + } + if (UtilMethods.isSet(reindexWorking)) { + result.put(IndexType.REINDEX_WORKING, reindexWorking); + } + if (UtilMethods.isSet(siteSearch)) { + result.put(IndexType.SITE_SEARCH, siteSearch); + } + + return result; + } + + // Modern helper methods + + /** + * Get index name for a specific type + */ + public String getIndexName(IndexType indexType) { + switch (indexType) { + case LIVE: return live; + case WORKING: return working; + case REINDEX_LIVE: return reindexLive; + case REINDEX_WORKING: return reindexWorking; + case SITE_SEARCH: return siteSearch; + default: return null; + } + } + + /** + * Create a new version of this indices configuration + */ + public IndicesInfoImpl withVersion(String newVersion) { + return toBuilder() + .version(newVersion) + .build(); + } + + /** + * Check if index name is modern format (includes version) + */ + public static boolean isModernIndexName(String indexName) { + return UtilMethods.isSet(indexName) && indexName.contains("_v") && + indexName.matches(".*_v\\d+$"); + } + + // Private helper methods + + private Optional getIndexNameForType(IndexType indexType) { + switch (indexType) { + case LIVE: return Optional.of(live); + case WORKING: return Optional.of(working); + case REINDEX_LIVE: return Optional.of(reindexLive); + case REINDEX_WORKING: return Optional.of(reindexWorking); + case SITE_SEARCH: return Optional.of(siteSearch); + + } + return Optional.empty(); + } + + private String extractTimestampFromIndexName(String indexName) { + if (isModernIndexName(indexName)) { + // Modern format: cluster_id.prefix_timestamp_vVersion + String withoutVersion = indexName.substring(0, indexName.lastIndexOf("_v")); + return withoutVersion.substring(withoutVersion.lastIndexOf("_") + 1); + } else { + // Legacy format: cluster_id.prefix_timestamp + return indexName.substring(indexName.lastIndexOf("_") + 1); + } + } + + // Object methods + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + + IndicesInfoImpl that = (IndicesInfoImpl) obj; + + return Objects.equals(version, that.version) && + Objects.equals(live, that.live) && + Objects.equals(working, that.working) && + Objects.equals(reindexLive, that.reindexLive) && + Objects.equals(reindexWorking, that.reindexWorking) && + Objects.equals(siteSearch, that.siteSearch); + } + + @Override + public int hashCode() { + return Objects.hash(live, working, reindexLive, reindexWorking, siteSearch, version); + } + + @Override + public String toString() { + return "ModernIndicesInfo{" + + "live='" + live + '\'' + + ", working='" + working + '\'' + + ", reindexLive='" + reindexLive + '\'' + + ", reindexWorking='" + reindexWorking + '\'' + + ", siteSearch='" + siteSearch + '\'' + + ", version=" + version + + '}'; + } + + /** + * Builder pattern for ModernIndicesInfo + */ + public static final class Builder { + private String live; + private String working; + private String reindexLive; + private String reindexWorking; + private String siteSearch; + private String version = OPEN_SEARCH_VERSION; // Default version + + public Builder() {} + + public Builder live(String live) { + this.live = live; + return this; + } + + public Builder working(String working) { + this.working = working; + return this; + } + + public Builder reindexLive(String reindexLive) { + this.reindexLive = reindexLive; + return this; + } + + public Builder reindexWorking(String reindexWorking) { + this.reindexWorking = reindexWorking; + return this; + } + + public Builder siteSearch(String siteSearch) { + this.siteSearch = siteSearch; + return this; + } + + public Builder version(String version) { + this.version = version; + return this; + } + + public Builder withNewIndicesName(IndexType... indexTypes) { + final String timestamp = simpleDateFormat.format(new Date()); + + for (IndexType indexType : indexTypes) { + String indexName = String.format( + INDEX_NAME_PATTERN, + ClusterFactory.getClusterId(), + indexType.getPrefix(), + timestamp, + version + ); + + // Update builder with a new index name + switch (indexType) { + case LIVE: + live(indexName); + break; + case WORKING: + working(indexName); + break; + case REINDEX_LIVE: + reindexLive(indexName); + break; + case REINDEX_WORKING: + reindexWorking(indexName); + break; + case SITE_SEARCH: + siteSearch(indexName); + break; + default: + Logger.warn(IndicesInfoImpl.class, "Unknown index type: " + indexType); + } + } + return this; + } + + /** + * Build the ModernIndicesInfo instance + */ + public IndicesInfoImpl build() { + return new IndicesInfoImpl(this); + } + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchBulkHelper.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchBulkHelper.java new file mode 100644 index 000000000000..bd6c0193e234 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchBulkHelper.java @@ -0,0 +1,343 @@ +package com.dotcms.content.opensearch.business; + +import com.dotcms.content.opensearch.util.OpenSearchDefaultClientProvider; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.common.reindex.ReindexEntry; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.portlets.contentlet.model.IndexPolicy; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.UtilMethods; +import io.vavr.control.Try; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import org.opensearch.client.opensearch.core.bulk.DeleteOperation; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; + +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Default; +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Helper class for handling OpenSearch bulk operations using ReindexEntry objects. + * This class provides a convenient interface to register documents for bulk processing + * and execute batch operations for both indexing and deletion. + * + * Usage pattern: + * 1. Create instance + * 2. Register documents using addToQueue() + * 3. Execute bulk operation using executeBulk() + * 4. Handle results and cleanup using clear() + * + * @author fabrizio + */ +@ApplicationScoped +@Default +public class OpenSearchBulkHelper { + + private static final int DEFAULT_BULK_TIMEOUT_MS = Config.getIntProperty("OPENSEARCH_BULK_TIMEOUT", 30000); + private static final int DEFAULT_BATCH_SIZE = Config.getIntProperty("OPENSEARCH_BULK_BATCH_SIZE", 100); + private static final String DEFAULT_INDEX_SUFFIX = Config.getStringProperty("ES_INDEX_NAME", "content"); + + @Inject + private OpenSearchDefaultClientProvider clientProvider; + + // Thread-safe collections for bulk operations + private final List operations = new ArrayList<>(); + private final Map entryMap = new ConcurrentHashMap<>(); + private final AtomicInteger operationCount = new AtomicInteger(0); + + /** + * Registers a ReindexEntry for bulk processing. + * The operation type (index/delete) is determined by the ReindexEntry.isDelete() flag. + * + * @param entry The ReindexEntry to process + * @throws DotDataException if there's an error preparing the operation + */ + public void addToQueue(ReindexEntry entry) throws DotDataException { + if (entry == null || !UtilMethods.isSet(entry.getIdentToIndex())) { + Logger.warn(this, "Invalid ReindexEntry provided - skipping"); + return; + } + + try { + BulkOperation operation = createBulkOperation(entry); + synchronized (operations) { + operations.add(operation); + entryMap.put(entry.getIdentToIndex(), entry); + operationCount.incrementAndGet(); + } + + Logger.debug(this, () -> String.format("Added %s operation for identifier: %s", + entry.isDelete() ? "DELETE" : "INDEX", entry.getIdentToIndex())); + + } catch (Exception e) { + Logger.error(this, "Error adding ReindexEntry to bulk queue: " + e.getMessage(), e); + throw new DotDataException("Failed to add entry to bulk queue", e); + } + } + + /** + * Registers multiple ReindexEntries for bulk processing. + * + * @param entries List of ReindexEntry objects to process + * @throws DotDataException if there's an error preparing any operation + */ + public void addToQueue(List entries) throws DotDataException { + if (entries == null || entries.isEmpty()) { + Logger.debug(this, "No entries provided to add to bulk queue"); + return; + } + + for (ReindexEntry entry : entries) { + addToQueue(entry); + } + } + + /** + * Executes the bulk operation with all registered documents. + * + * @return BulkOperationResult containing success/failure information + * @throws DotDataException if the bulk operation fails + */ + public BulkOperationResult executeBulk() throws DotDataException { + if (operations.isEmpty()) { + Logger.debug(this, "No operations to execute"); + return new BulkOperationResult(0, 0, new ArrayList<>(), new ArrayList<>()); + } + + try { + Logger.info(this, String.format("Executing bulk operation with %d operations", operationCount.get())); + + BulkRequest.Builder requestBuilder = new BulkRequest.Builder() + .operations(new ArrayList<>(operations)) + .timeout(Time.of(t -> t.time(String.valueOf(DEFAULT_BULK_TIMEOUT_MS) + "ms"))); + + OpenSearchClient client = clientProvider.getClient(); + BulkResponse response = client.bulk(requestBuilder.build()); + + return processBulkResponse(response); + + } catch (IOException e) { + Logger.error(this, "OpenSearch bulk operation failed: " + e.getMessage(), e); + throw new DotDataException("Bulk operation execution failed", e); + } catch (Exception e) { + Logger.error(this, "Unexpected error during bulk operation: " + e.getMessage(), e); + throw new DotRuntimeException("Unexpected bulk operation error", e); + } + } + + /** + * Returns the current number of operations in the queue. + * + * @return number of pending operations + */ + public int getQueueSize() { + return operationCount.get(); + } + + /** + * Checks if the queue has reached the configured batch size. + * + * @return true if queue size >= batch size + */ + public boolean isBatchReady() { + return getQueueSize() >= DEFAULT_BATCH_SIZE; + } + + /** + * Clears all registered operations and resets internal state. + * Should be called after successful bulk execution or for cleanup. + */ + public void clear() { + synchronized (operations) { + operations.clear(); + entryMap.clear(); + operationCount.set(0); + } + Logger.debug(this, "Bulk operation queue cleared"); + } + + /** + * Creates a BulkOperation from a ReindexEntry. + * Determines operation type based on the entry's delete flag. + */ + private BulkOperation createBulkOperation(ReindexEntry entry) throws DotDataException { + String identifier = entry.getIdentToIndex(); + + if (entry.isDelete()) { + return createDeleteOperation(identifier); + } else { + return createIndexOperation(entry); + } + } + + /** + * Creates a delete operation for the given identifier. + */ + private BulkOperation createDeleteOperation(String identifier) { + return BulkOperation.of(op -> op.delete(DeleteOperation.of(del -> del + .index(getIndexName()) + .id(identifier) + ))); + } + + /** + * Creates an index operation for the given ReindexEntry. + * Retrieves the contentlet and converts it to the document format. + */ + private BulkOperation createIndexOperation(ReindexEntry entry) throws DotDataException { + String identifier = entry.getIdentToIndex(); + + try { + // Retrieve the contentlet for indexing + Contentlet contentlet = APILocator.getContentletAPI() + .findContentletByIdentifier(identifier, false, APILocator.getLanguageAPI().getDefaultLanguage().getId(), + APILocator.systemUser(), false); + + if (contentlet == null || !UtilMethods.isSet(contentlet.getIdentifier())) { + throw new DotDataException("Contentlet not found for identifier: " + identifier); + } + + // Convert contentlet to indexable document + Map document = createDocumentFromContentlet(contentlet); + + return BulkOperation.of(op -> op.index(IndexOperation.of(idx -> idx + .index(getIndexName()) + .id(identifier) + .document(document) + ))); + + } catch (Exception e) { + Logger.error(this, "Error creating index operation for identifier: " + identifier, e); + throw new DotDataException("Failed to create index operation", e); + } + } + + /** + * Converts a Contentlet to a Map for OpenSearch indexing. + * Uses the existing ContentletIndexAPI transformation logic. + */ + private Map createDocumentFromContentlet(Contentlet contentlet) throws DotDataException { + try { + return Map.of(); + /* + APILocator.getContentletIndexAPI() + .toMap(contentlet, IndexPolicy.WAIT_FOR, false); + */ + } catch (Exception e) { + Logger.error(this, "Error converting contentlet to document: " + e.getMessage(), e); + throw new DotDataException("Failed to convert contentlet to document", e); + } + } + + /** + * Processes the BulkResponse and creates a result object with success/failure details. + */ + private BulkOperationResult processBulkResponse(BulkResponse response) { + List successful = new ArrayList<>(); + List errors = new ArrayList<>(); + + int successCount = 0; + int errorCount = 0; + + for (BulkResponseItem item : response.items()) { + if (item.error() != null) { + errorCount++; + String identifier = item.id(); + ReindexEntry entry = entryMap.get(identifier); + + errors.add(new BulkOperationError( + identifier, + item.error().type(), + item.error().reason(), + entry + )); + + Logger.error(this, String.format("Bulk operation failed for %s: %s - %s", + identifier, item.error().type(), item.error().reason())); + } else { + successCount++; + successful.add(item.id()); + Logger.debug(this, () -> "Bulk operation successful for: " + item.id()); + } + } + + Logger.info(this, String.format("Bulk operation completed - Success: %d, Errors: %d", + successCount, errorCount)); + + return new BulkOperationResult(successCount, errorCount, successful, errors); + } + + /** + * Gets the current index name. + * This should be enhanced to use the proper index resolution logic. + */ + private String getIndexName() { + // TODO: Use proper index resolution logic from ContentletIndexAPI + return Try.of(() -> APILocator.getContentletIndexAPI().getActiveIndexName(DEFAULT_INDEX_SUFFIX)) + .getOrElse(() -> { + Logger.warn(this, "Could not resolve active index name, using default"); + return "dotcms_" + DEFAULT_INDEX_SUFFIX; + }); + } + + /** + * Result class containing bulk operation execution details. + */ + public static class BulkOperationResult { + private final int successCount; + private final int errorCount; + private final List successful; + private final List errors; + + public BulkOperationResult(int successCount, int errorCount, + List successful, List errors) { + this.successCount = successCount; + this.errorCount = errorCount; + this.successful = successful; + this.errors = errors; + } + + public int getSuccessCount() { return successCount; } + public int getErrorCount() { return errorCount; } + public List getSuccessful() { return successful; } + public List getErrors() { return errors; } + public boolean hasErrors() { return errorCount > 0; } + public int getTotalOperations() { return successCount + errorCount; } + } + + /** + * Error information for failed bulk operations. + */ + public static class BulkOperationError { + private final String identifier; + private final String errorType; + private final String errorReason; + private final ReindexEntry entry; + + public BulkOperationError(String identifier, String errorType, String errorReason, ReindexEntry entry) { + this.identifier = identifier; + this.errorType = errorType; + this.errorReason = errorReason; + this.entry = entry; + } + + public String getIdentifier() { return identifier; } + public String getErrorType() { return errorType; } + public String getErrorReason() { return errorReason; } + public ReindexEntry getEntry() { return entry; } + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchIndexAPI.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchIndexAPI.java new file mode 100644 index 000000000000..d86b42944895 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchIndexAPI.java @@ -0,0 +1,97 @@ +package com.dotcms.content.opensearch.business; + +import com.dotmarketing.business.DotStateException; +import com.dotmarketing.exception.DotDataException; +import java.io.IOException; +import java.util.Map; +import org.opensearch.client.opensearch.indices.CreateIndexResponse; + +/** + * OpenSearch Index API interface for managing OpenSearch indices operations. + * This is the OpenSearch equivalent of ESIndexAPI for OpenSearch 3.7. + * + * @author fabrizio + */ +public interface OpenSearchIndexAPI { + + /** + * Checks if an index exists in OpenSearch + * + * @param indexName the name of the index to check + * @return true if the index exists, false otherwise + */ + boolean indexExists(String indexName); + + /** + * Creates an index with default settings + * + * @param indexName the name of the index to create + * @throws DotStateException if there's a state error + * @throws IOException if there's an I/O error + */ + void createIndex(String indexName) throws DotStateException, IOException; + + /** + * Creates an index with default settings and specific number of shards + * If shards < 1 then default shards will be used + * + * @param indexName the name of the index to create + * @param shards number of shards for the index + * @return CreateIndexResponse with the response details + * @throws DotStateException if there's a state error + * @throws IOException if there's an I/O error + */ + CreateIndexResponse createIndex(String indexName, int shards) throws DotStateException, IOException; + + /** + * Creates an index with custom settings and specific number of shards + * If settings is null, default settings will be applied + * If shards < 1, then default shards will be used + * + * @param indexName the name of the index to create + * @param settings custom JSON settings for the index + * @param shards number of shards for the index + * @return CreateIndexResponse with the response details + * @throws DotStateException if there's a state error + * @throws IOException if there's an I/O error + */ + CreateIndexResponse createIndex(String indexName, String settings, int shards) throws DotStateException, IOException; + + /** + * Gets the default index settings as JSON string + * + * @return JSON string with default index settings + */ + String getDefaultIndexSettings(); + + /** + * Returns indices statistics + * + * @return map of index names to their statistics + */ + Map getIndicesStats(); + + /** + * Deletes an index by name + * + * @param indexName the name of the index to delete + * @return true if the deletion was acknowledged, false otherwise + */ + boolean delete(String indexName); + + /** + * Given an index name, returns the full name including the cluster id prefix + * + * @param name index name + * @return index name with cluster id prefix + */ + String getNameWithClusterIDPrefix(String name); + + /** + * Given an index name with cluster prefix, returns the name without the prefix + * + * @param name index name with cluster prefix + * @return index name without cluster prefix + */ + String removeClusterIdFromName(String name); +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchIndexAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchIndexAPIImpl.java new file mode 100644 index 000000000000..714604447e86 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/business/OpenSearchIndexAPIImpl.java @@ -0,0 +1,230 @@ +package com.dotcms.content.opensearch.business; + +import static com.dotcms.content.elasticsearch.business.LegacyIndicesInfo.CLUSTER_PREFIX; + +import com.dotcms.cdi.CDIUtils; +import com.dotcms.content.opensearch.util.OpenSearchDefaultClientProvider; +import com.dotcms.enterprise.cluster.ClusterFactory; +import com.dotcms.rest.api.v1.DotObjectMapperProvider; +import com.dotmarketing.business.DotStateException; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.AdminLogger; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.UtilMethods; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.rainerhahnekamp.sneakythrow.Sneaky; +import io.vavr.Lazy; +import io.vavr.control.Try; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.time.Duration; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Default; +import javax.inject.Inject; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.CreateIndexResponse; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.ExistsRequest; +import org.opensearch.client.opensearch.indices.IndexSettings; + +/** + * Implementation of OpenSearchIndexAPI for managing OpenSearch indices operations. + * This is the OpenSearch equivalent of ESIndexAPI for OpenSearch 3.7. + * + * @author fabrizio + */ +@ApplicationScoped +@Default +public class OpenSearchIndexAPIImpl implements OpenSearchIndexAPI { + + @Inject + OpenSearchDefaultClientProvider clientProvider; + + private static ObjectMapper objectMapper = DotObjectMapperProvider.createDefaultMapper(); + + public static final int INDEX_OPERATIONS_TIMEOUT_IN_MS = + Config.getIntProperty("OPENSEARCH_INDEX_OPERATIONS_TIMEOUT", 15000); + + private final Lazy clusterPrefix; + + OpenSearchIndexAPIImpl(Lazy clusterPrefix) { + this.clusterPrefix = clusterPrefix; + } + + public OpenSearchIndexAPIImpl() { + this(Lazy.of(() -> CLUSTER_PREFIX + ClusterFactory.getClusterId() + ".")); + } + + @Override + public boolean indexExists(String indexName) { + if (indexName == null) { + return false; + } + + try { + final ExistsRequest request = ExistsRequest.of(builder -> + builder.index(getNameWithClusterIDPrefix(indexName)) + ); + + return clientProvider.getClient().indices().exists(request).value(); + } catch (Exception e) { + Logger.error(this.getClass(), "Error checking if index exists: " + indexName, e); + return false; + } + } + + @Override + public void createIndex(String indexName) throws DotStateException, IOException { + createIndex(indexName, null, 0); + } + + @Override + public CreateIndexResponse createIndex(String indexName, int shards) throws DotStateException, IOException { + return createIndex(indexName, null, shards); + } + + @Override + public CreateIndexResponse createIndex(final String indexName, String settings, int shards) + throws DotStateException, IOException { + + AdminLogger.log(this.getClass(), "createIndex", + "Trying to create index: " + indexName + " with shards: " + shards); + + shards = shards > 0 ? shards : Config.getIntProperty("opensearch.index.number_of_shards", 1); + + Map settingsMap = (settings == null) ? new HashMap<>() : + objectMapper.readValue(settings, LinkedHashMap.class); + + settingsMap.put("index.number_of_shards", shards); + settingsMap.put("index.auto_expand_replicas", "0-all"); + settingsMap.putIfAbsent("index.mapping.total_fields.limit", 10000); + settingsMap.putIfAbsent("index.mapping.nested_fields.limit", 10000); + settingsMap.putIfAbsent("index.query.default_field", + Config.getStringProperty("OPENSEARCH_INDEX_QUERY_DEFAULT_FIELD", "catchall")); + + final CreateIndexRequest request = CreateIndexRequest.of(builder -> + builder.index(getNameWithClusterIDPrefix(indexName)) + .settings(IndexSettings.of(settingsBuilder -> { + settingsMap.forEach((key, value) -> { + if (value instanceof Integer) { + settingsBuilder.numberOfShards((Integer) value); + } else if (value instanceof String) { + settingsBuilder.autoExpandReplicas((String) value); + } + // Add more specific mappings as needed + }); + return settingsBuilder; + })) + .timeout(Time.of(timeBuilder -> + timeBuilder.time(String.format("%dms",INDEX_OPERATIONS_TIMEOUT_IN_MS)) + )) + ); + + try { + final CreateIndexResponse response = clientProvider.getClient().indices().create(request); + AdminLogger.log(this.getClass(), "createIndex", + "Index created: " + indexName + " with shards: " + shards); + return response; + } catch (Exception e) { + Logger.error(this.getClass(), "Error creating index: " + indexName, e); + throw new DotStateException("Failed to create index: " + indexName, e); + } + } + + @Override + public String getDefaultIndexSettings() { + String settings = null; + try { + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final URL url = classLoader.getResource("opensearch-content-settings.json"); + if (url != null) { + settings = new String(com.liferay.util.FileUtil.getBytes(new File(url.getPath()))); + } + } catch (Exception e) { + Logger.error(this.getClass(), "Cannot load opensearch-content-settings.json file, using defaults", e); + } + + // Fallback to basic settings if file not found + if (settings == null) { + settings = "{\n" + + " \"number_of_shards\": 1,\n" + + " \"number_of_replicas\": 0,\n" + + " \"auto_expand_replicas\": \"0-all\",\n" + + " \"mapping\": {\n" + + " \"total_fields\": {\n" + + " \"limit\": 10000\n" + + " },\n" + + " \"nested_fields\": {\n" + + " \"limit\": 10000\n" + + " }\n" + + " },\n" + + " \"query\": {\n" + + " \"default_field\": \"catchall\"\n" + + " }\n" + + "}"; + } + + return settings; + } + + @Override + public Map getIndicesStats() { + // TODO: Implement indices stats retrieval using OpenSearch client + // This will require using the cluster stats or indices stats API + Logger.info(this.getClass(), "getIndicesStats not yet implemented for OpenSearch"); + return new HashMap<>(); + } + + @Override + public boolean delete(String indexName) { + if (indexName == null) { + Logger.error(this.getClass(), "Failed to delete a null OpenSearch index"); + return true; + } + + try { + final DeleteIndexRequest request = DeleteIndexRequest.of(builder -> + builder.index(getNameWithClusterIDPrefix(indexName)) + .timeout(Time.of(timeBuilder -> + timeBuilder.time(Duration.ofMillis(INDEX_OPERATIONS_TIMEOUT_IN_MS).toString()) + )) + ); + + final var response = clientProvider.getClient().indices().delete(request); + return response.acknowledged(); + } catch (Exception e) { + Logger.error(this.getClass(), "Error deleting index: " + indexName, e); + throw new RuntimeException("Failed to delete index: " + indexName, e); + } + } + + @Override + public String getNameWithClusterIDPrefix(final String name) { + return hasClusterPrefix(name) ? name : clusterPrefix.get() + name; + } + + @Override + public String removeClusterIdFromName(final String name) { + if (name == null) return ""; + return name.indexOf(".") > -1 + ? name.substring(name.lastIndexOf(".") + 1, name.length()) + : name; + } + + /** + * Checks if the given index name has the cluster prefix + */ + boolean hasClusterPrefix(final String indexName) { + return indexName != null && indexName.startsWith(clusterPrefix.get()); + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/util/ConfigurableOpenSearchProvider.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/ConfigurableOpenSearchProvider.java new file mode 100644 index 000000000000..0a61360f357b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/ConfigurableOpenSearchProvider.java @@ -0,0 +1,347 @@ +package com.dotcms.content.opensearch.util; + +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.UtilMethods; +import java.net.URI; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.ssl.TrustSelfSignedStrategy; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.ssl.SSLContexts; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.GeneralSecurityException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.List; + +/** + * Internal configurable OpenSearch client provider implementation. + * Package-private class used internally by OpenSearchClients and for test configurations. + * Provides a straightforward way to configure and create OpenSearch clients. + */ +class ConfigurableOpenSearchProvider { + + private static final String OS_ENDPOINTS = "OS_ENDPOINTS"; + private static final String OS_HOSTNAME = "OS_HOSTNAME"; + private static final String OS_PROTOCOL = "OS_PROTOCOL"; + private static final String OS_PORT = "OS_PORT"; + + private static final String OS_AUTH_TYPE = "OS_AUTH_TYPE"; + private static final String OS_AUTH_BASIC_USER = "OS_AUTH_BASIC_USER"; + private static final String OS_AUTH_BASIC_PASSWORD = "OS_AUTH_BASIC_PASSWORD"; + private static final String OS_AUTH_JWT_TOKEN = "OS_AUTH_JWT_TOKEN"; + + private static final String OS_TLS_ENABLED = "OS_TLS_ENABLED"; + private static final String OS_TLS_TRUST_SELF_SIGNED = "OS_TLS_TRUST_SELF_SIGNED"; + private static final String OS_TLS_CLIENT_CERT = "OS_TLS_CLIENT_CERT"; + private static final String OS_TLS_CLIENT_KEY = "OS_TLS_CLIENT_KEY"; + private static final String OS_TLS_CA_CERT = "OS_TLS_CA_CERT"; + + private static final String OS_CONNECTION_TIMEOUT = "OS_CONNECTION_TIMEOUT"; + private static final String OS_SOCKET_TIMEOUT = "OS_SOCKET_TIMEOUT"; + private static final String OS_MAX_CONNECTIONS = "OS_MAX_CONNECTIONS"; + private static final String OS_MAX_CONNECTIONS_PER_ROUTE = "OS_MAX_CONNECTIONS_PER_ROUTE"; + + private static final String BASIC_AUTH_TYPE = "BASIC"; + private static final String JWT_AUTH_TYPE = "JWT"; + private static final String CERT_AUTH_TYPE = "CERT"; + + private static final String HTTPS_PROTOCOL = "https"; + private static final String HTTP_PROTOCOL = "http"; + + private OpenSearchClient client; + private OpenSearchTransport transport; + + /** + * Create provider using configuration from properties + */ + public ConfigurableOpenSearchProvider() { + buildClient(); + } + + /** + * Create provider using custom configuration + */ + public ConfigurableOpenSearchProvider(OpenSearchClientConfig config) { + buildClient(config); + } + + /** + * Build client using properties configuration + */ + private void buildClient() { + try { + OpenSearchClientConfig config = loadConfigurationFromProperties(); + buildClient(config); + } catch (Exception e) { + Logger.error(this.getClass(), "Error building OpenSearch client from properties", e); + throw new DotRuntimeException("Failed to build OpenSearch client", e); + } + } + + /** + * Build client using provided configuration + */ + private void buildClient(OpenSearchClientConfig config) { + try { + transport = createTransport(config); + client = new OpenSearchClient(transport); + + Logger.info(this.getClass(), "OpenSearch client initialized successfully with endpoints: " + config.endpoints()); + } catch (Exception e) { + Logger.error(this.getClass(), "Error building OpenSearch client", e); + throw new DotRuntimeException("Failed to build OpenSearch client", e); + } + } + + /** + * Load configuration from dotCMS properties + */ + private OpenSearchClientConfig loadConfigurationFromProperties() { + ImmutableOpenSearchClientConfig.Builder builder = OpenSearchClientConfig.builder(); + + // Load endpoints + String[] endpoints = Config.getStringArrayProperty(OS_ENDPOINTS, getDefaultEndpoints()); + builder.endpoints(Arrays.asList(endpoints)); + + // Load authentication settings + String authType = Config.getStringProperty(OS_AUTH_TYPE, BASIC_AUTH_TYPE); + + if (BASIC_AUTH_TYPE.equals(authType)) { + String username = Config.getStringProperty(OS_AUTH_BASIC_USER, null); + String password = Config.getStringProperty(OS_AUTH_BASIC_PASSWORD, null); + if (UtilMethods.isSet(username) && UtilMethods.isSet(password)) { + builder.username(username).password(password); + } + } else if (JWT_AUTH_TYPE.equals(authType)) { + String token = Config.getStringProperty(OS_AUTH_JWT_TOKEN, null); + if (UtilMethods.isSet(token)) { + builder.jwtToken(token); + } + } else if (CERT_AUTH_TYPE.equals(authType)) { + String clientCert = Config.getStringProperty(OS_TLS_CLIENT_CERT, null); + String clientKey = Config.getStringProperty(OS_TLS_CLIENT_KEY, null); + if (UtilMethods.isSet(clientCert) && UtilMethods.isSet(clientKey)) { + builder.clientCertPath(clientCert).clientKeyPath(clientKey); + } + } + + // Load TLS settings + boolean tlsEnabled = Config.getBooleanProperty(OS_TLS_ENABLED, false); + builder.tlsEnabled(tlsEnabled); + + if (tlsEnabled) { + builder.trustSelfSigned(Config.getBooleanProperty(OS_TLS_TRUST_SELF_SIGNED, false)); + String caCert = Config.getStringProperty(OS_TLS_CA_CERT, null); + if (UtilMethods.isSet(caCert)) { + builder.caCertPath(caCert); + } + } + + // Load connection settings with defaults + int connectionTimeout = Config.getIntProperty(OS_CONNECTION_TIMEOUT, 10000); + int socketTimeout = Config.getIntProperty(OS_SOCKET_TIMEOUT, 30000); + int maxConnections = Config.getIntProperty(OS_MAX_CONNECTIONS, 100); + int maxConnectionsPerRoute = Config.getIntProperty(OS_MAX_CONNECTIONS_PER_ROUTE, 50); + + builder.connectionTimeout(java.time.Duration.ofMillis(connectionTimeout)) + .socketTimeout(java.time.Duration.ofMillis(socketTimeout)) + .maxConnections(maxConnections) + .maxConnectionsPerRoute(maxConnectionsPerRoute); + + return builder.build(); + } + + /** + * Get default endpoints if not configured + */ + private String[] getDefaultEndpoints() { + String hostname = Config.getStringProperty(OS_HOSTNAME, "localhost"); + String protocol = Config.getStringProperty(OS_PROTOCOL, HTTPS_PROTOCOL); + int port = Config.getIntProperty(OS_PORT, 9200); + + return new String[] { protocol + "://" + hostname + ":" + port }; + } + + /** + * Create OpenSearch transport based on configuration + */ + private OpenSearchTransport createTransport(OpenSearchClientConfig config) { + HttpHost[] hosts = createHttpHosts(config.endpoints()); + + ApacheHttpClient5TransportBuilder builder = ApacheHttpClient5TransportBuilder.builder(hosts); + + // Configure HTTP client + builder.setHttpClientConfigCallback(httpClientBuilder -> { + try { + configureAuthentication(httpClientBuilder, config); + configureTLS(httpClientBuilder, config); + configureTimeouts(httpClientBuilder, config); + return httpClientBuilder; + } catch (Exception e) { + Logger.error(this.getClass(), "Error configuring HTTP client", e); + throw new DotRuntimeException("Failed to configure HTTP client", e); + } + }); + + // Set JSON mapper + builder.setMapper(new JacksonJsonpMapper()); + + return builder.build(); + } + + /** + * Convert endpoint strings to HttpHost array + */ + private HttpHost[] createHttpHosts(List endpoints) { + return endpoints.stream().map(endpoint -> { + try { + URL url = URI.create(endpoint).toURL(); + return new HttpHost(url.getProtocol(), url.getHost(), url.getPort()); + } catch (MalformedURLException e) { + Logger.error(this.getClass(), "Invalid endpoint URL: " + endpoint, e); + throw new DotRuntimeException("Invalid endpoint URL: " + endpoint, e); + } + }).toArray(HttpHost[]::new); + } + + /** + * Configure authentication for HTTP client + */ + private void configureAuthentication(org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder httpClientBuilder, + OpenSearchClientConfig config) { + // Basic authentication + if (config.username().isPresent() && config.password().isPresent()) { + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(new AuthScope(null, -1), + new UsernamePasswordCredentials(config.username().get(), config.password().get().toCharArray())); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + Logger.info(this.getClass(), "OpenSearch client configured with basic authentication"); + } + + // JWT authentication will be handled via headers in the request interceptor + if (config.jwtToken().isPresent()) { + httpClientBuilder.addRequestInterceptorLast((request, entity, context) -> { + request.setHeader("Authorization", "Bearer " + config.jwtToken().get()); + }); + Logger.info(this.getClass(), "OpenSearch client configured with JWT authentication"); + } + } + + /** + * Configure TLS for HTTP client + */ + private void configureTLS(org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder httpClientBuilder, + OpenSearchClientConfig config) throws GeneralSecurityException { + if (!config.tlsEnabled()) { + return; + } + + SSLContext sslContext; + + if (config.clientCertPath().isPresent() && config.clientKeyPath().isPresent()) { + // Certificate-based authentication (would need PEM reader implementation) + Logger.warn(this.getClass(), "Certificate-based TLS not yet implemented, using trust-self-signed strategy"); + sslContext = createTrustSelfSignedSSLContext(); + } else if (config.trustSelfSigned()) { + sslContext = createTrustSelfSignedSSLContext(); + } else { + sslContext = SSLContext.getDefault(); + } + + // For HttpClient5, SSL is configured via the connection manager + // This is a simplified approach - SSL context will be used by default connection manager + Logger.info(this.getClass(), "OpenSearch client configured with TLS"); + } + + /** + * Create SSL context that trusts self-signed certificates + */ + private SSLContext createTrustSelfSignedSSLContext() throws GeneralSecurityException { + try { + return SSLContexts.custom() + .loadTrustMaterial(new TrustSelfSignedStrategy()) + .build(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + Logger.error(this.getClass(), "Error creating trust-self-signed SSL context", e); + throw new GeneralSecurityException("Failed to create SSL context", e); + } + } + + /** + * Configure timeouts for HTTP client + */ + private void configureTimeouts(org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder httpClientBuilder, + OpenSearchClientConfig config) { + org.apache.hc.client5.http.config.RequestConfig requestConfig = + org.apache.hc.client5.http.config.RequestConfig.custom() + .setConnectTimeout(org.apache.hc.core5.util.Timeout.ofMilliseconds(config.connectionTimeout().toMillis())) + .setResponseTimeout(org.apache.hc.core5.util.Timeout.ofMilliseconds(config.socketTimeout().toMillis())) + .build(); + + httpClientBuilder.setDefaultRequestConfig(requestConfig); + + // Connection pool configuration for async client + org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder connManagerBuilder = + org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnTotal(config.maxConnections()) + .setMaxConnPerRoute(config.maxConnectionsPerRoute()); + + httpClientBuilder.setConnectionManager(connManagerBuilder.build()); + } + + /** + * Get the OpenSearch client instance + */ + public OpenSearchClient getClient() { + return client; + } + + /** + * Close the client and transport + */ + public void close() throws IOException { + if (transport != null) { + transport.close(); + } + } + + /** + * Rebuild the client (useful for configuration changes) + */ + public void rebuildClient() { + Logger.info(this.getClass(), "Rebuilding OpenSearch client"); + try { + close(); + } catch (IOException e) { + Logger.warn(this.getClass(), "Error closing existing client during rebuild", e); + } + buildClient(); + } + + /** + * Rebuild the client with new configuration + */ + public void rebuildClient(OpenSearchClientConfig config) { + Logger.info(this.getClass(), "Rebuilding OpenSearch client with new configuration"); + try { + close(); + } catch (IOException e) { + Logger.warn(this.getClass(), "Error closing existing client during rebuild", e); + } + buildClient(config); + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OPENSEARCH_PROPERTIES.md b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OPENSEARCH_PROPERTIES.md new file mode 100644 index 000000000000..c58d11a6c2b2 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OPENSEARCH_PROPERTIES.md @@ -0,0 +1,141 @@ +# OpenSearch Client Configuration Properties + +This document describes all available configuration properties for the DotOpenSearchClientProvider. + +## Connection Properties + +### Endpoints Configuration +```properties +# Multiple endpoints (comma-separated) +OS_ENDPOINTS=https://node1.opensearch.com:9200,https://node2.opensearch.com:9200 + +# OR individual endpoint components (when OS_ENDPOINTS is not set) +OS_HOSTNAME=localhost +OS_PROTOCOL=https +OS_PORT=9200 +``` + +## Authentication Properties + +### Basic Authentication +```properties +OS_AUTH_TYPE=BASIC +OS_AUTH_BASIC_USER=admin +OS_AUTH_BASIC_PASSWORD=password123 +``` + +### JWT Token Authentication +```properties +OS_AUTH_TYPE=JWT +OS_AUTH_JWT_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9... +``` + +### Certificate Authentication +```properties +OS_AUTH_TYPE=CERT +OS_TLS_CLIENT_CERT=/path/to/client.pem +OS_TLS_CLIENT_KEY=/path/to/client.key +``` + +## TLS/SSL Properties + +### Enable TLS +```properties +OS_TLS_ENABLED=true +OS_TLS_TRUST_SELF_SIGNED=false +OS_TLS_CA_CERT=/path/to/ca-cert.pem +``` + +## Connection Pool Properties + +### Timeouts (in milliseconds) +```properties +OS_CONNECTION_TIMEOUT=10000 # 10 seconds +OS_SOCKET_TIMEOUT=30000 # 30 seconds +``` + +### Connection Limits +```properties +OS_MAX_CONNECTIONS=100 +OS_MAX_CONNECTIONS_PER_ROUTE=50 +``` + +## Usage Examples + +### Local Development (HTTP) +```properties +OS_HOSTNAME=localhost +OS_PROTOCOL=http +OS_PORT=9200 +OS_AUTH_TYPE=BASIC +OS_AUTH_BASIC_USER=admin +OS_AUTH_BASIC_PASSWORD=admin +``` + +### Production HTTPS with Basic Auth +```properties +OS_ENDPOINTS=https://opensearch.prod.com:9200 +OS_AUTH_TYPE=BASIC +OS_AUTH_BASIC_USER=prod_user +OS_AUTH_BASIC_PASSWORD=secure_password +OS_TLS_ENABLED=true +OS_TLS_TRUST_SELF_SIGNED=false +``` + +### Cluster with JWT Authentication +```properties +OS_ENDPOINTS=https://os-node1:9200,https://os-node2:9200,https://os-node3:9200 +OS_AUTH_TYPE=JWT +OS_AUTH_JWT_TOKEN=your_jwt_token_here +OS_TLS_ENABLED=true +OS_TLS_TRUST_SELF_SIGNED=true +``` + +### Certificate-based Authentication +```properties +OS_ENDPOINTS=https://secure-opensearch:9200 +OS_AUTH_TYPE=CERT +OS_TLS_ENABLED=true +OS_TLS_CLIENT_CERT=/opt/certs/client.pem +OS_TLS_CLIENT_KEY=/opt/certs/client.key +OS_TLS_CA_CERT=/opt/certs/ca.pem +``` + +## Property Defaults + +| Property | Default Value | Description | +|----------|---------------|-------------| +| OS_HOSTNAME | localhost | OpenSearch hostname | +| OS_PROTOCOL | https | Protocol (http/https) | +| OS_PORT | 9200 | OpenSearch port | +| OS_AUTH_TYPE | BASIC | Authentication type | +| OS_TLS_ENABLED | false | Enable TLS/SSL | +| OS_TLS_TRUST_SELF_SIGNED | false | Trust self-signed certificates | +| OS_CONNECTION_TIMEOUT | 10000 | Connection timeout (ms) | +| OS_SOCKET_TIMEOUT | 30000 | Socket timeout (ms) | +| OS_MAX_CONNECTIONS | 100 | Max total connections | +| OS_MAX_CONNECTIONS_PER_ROUTE | 50 | Max connections per route | + +## Configuration Priority + +1. **OS_ENDPOINTS** - If set, overrides individual hostname/protocol/port +2. **OS_HOSTNAME/OS_PROTOCOL/OS_PORT** - Used when OS_ENDPOINTS is not set +3. Default values - Used when properties are not configured + +## Programmatic Configuration + +You can also configure the client programmatically using the builder pattern: + +```java +OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .endpoints("https://node1:9200", "https://node2:9200") + .username("admin") + .password("password") + .tlsEnabled(true) + .trustSelfSigned(false) + .connectionTimeout(Duration.ofSeconds(15)) + .build(); + +DotOpenSearchClientProvider provider = new DotOpenSearchClientProvider(config); +OpenSearchClient client = provider.getClient(); +``` \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OpenSearchClientConfig.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OpenSearchClientConfig.java new file mode 100644 index 000000000000..1553849d43ee --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OpenSearchClientConfig.java @@ -0,0 +1,146 @@ +package com.dotcms.content.opensearch.util; + +import com.dotmarketing.util.UtilMethods; +import org.immutables.value.Value; + +import javax.annotation.Nullable; +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +/** + * Configuration class for OpenSearch client using Builder pattern. + * Provides a simple and flexible way to configure OpenSearch client settings. + */ +@Value.Immutable +public abstract class OpenSearchClientConfig { + + /** + * List of OpenSearch endpoints (e.g., "https://localhost:9200") + */ + public abstract List endpoints(); + + /** + * Username for basic authentication + */ + public abstract Optional username(); + + /** + * Password for basic authentication + */ + public abstract Optional password(); + + /** + * JWT token for token-based authentication + */ + public abstract Optional jwtToken(); + + /** + * Path to client certificate for TLS authentication + */ + public abstract Optional clientCertPath(); + + /** + * Path to client private key for TLS authentication + */ + public abstract Optional clientKeyPath(); + + /** + * Path to CA certificate for TLS verification + */ + public abstract Optional caCertPath(); + + /** + * Whether TLS is enabled + */ + @Value.Default + public boolean tlsEnabled() { + return false; + } + + /** + * Whether to trust self-signed certificates + */ + @Value.Default + public boolean trustSelfSigned() { + return false; + } + + /** + * Connection timeout + */ + @Value.Default + public Duration connectionTimeout() { + return Duration.ofSeconds(10); + } + + /** + * Socket timeout + */ + @Value.Default + public Duration socketTimeout() { + return Duration.ofSeconds(30); + } + + /** + * Maximum number of connections + */ + @Value.Default + public int maxConnections() { + return 100; + } + + /** + * Maximum connections per route + */ + @Value.Default + public int maxConnectionsPerRoute() { + return 50; + } + + /** + * Create a new builder instance + */ + public static ImmutableOpenSearchClientConfig.Builder builder() { + return ImmutableOpenSearchClientConfig.builder(); + } + + /** + * Validation method to ensure configuration is valid + */ + @Value.Check + protected void check() { + if (endpoints().isEmpty()) { + throw new IllegalArgumentException("At least one endpoint must be specified"); + } + + for (String endpoint : endpoints()) { + if (!UtilMethods.isSet(endpoint)) { + throw new IllegalArgumentException("Endpoints cannot be null or empty"); + } + } + + // Validate TLS configuration + if (tlsEnabled()) { + boolean hasCertAuth = clientCertPath().isPresent() && clientKeyPath().isPresent(); + boolean hasBasicAuth = username().isPresent() && password().isPresent(); + boolean hasJwtAuth = jwtToken().isPresent(); + + if (!hasCertAuth && !hasBasicAuth && !hasJwtAuth && !trustSelfSigned()) { + throw new IllegalArgumentException( + "TLS is enabled but no authentication method or trust-self-signed is configured"); + } + } + + // Validate authentication methods are not conflicting + int authMethods = 0; + if (username().isPresent() || password().isPresent()) authMethods++; + if (jwtToken().isPresent()) authMethods++; + if (clientCertPath().isPresent() || clientKeyPath().isPresent()) authMethods++; + + if (authMethods > 1) { + throw new IllegalArgumentException( + "Only one authentication method should be configured (basic, JWT, or certificate)"); + } + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OpenSearchDefaultClientProvider.java b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OpenSearchDefaultClientProvider.java new file mode 100644 index 000000000000..9f4ec7746ca1 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/opensearch/util/OpenSearchDefaultClientProvider.java @@ -0,0 +1,114 @@ +package com.dotcms.content.opensearch.util; + +import com.dotmarketing.util.Logger; +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Default; +import org.opensearch.client.opensearch.OpenSearchClient; + +import java.io.IOException; + +/** + * Public immutable singleton for OpenSearch client access. + * Provides a single client instance throughout the application configured once at startup + * using default configuration from properties. + * + * This class uses the Initialization-on-demand holder idiom for thread-safe singleton + * creation. The singleton is immutable after initialization. + * + * For custom configurations in tests, use ConfigurableOpenSearchProvider directly + * with the OpenSearchClientConfig builder pattern. + * + * @author fabrizio + */ +@ApplicationScoped +@Default +public class OpenSearchDefaultClientProvider { + + /** + * Immutable internal provider - configured once at startup + */ + private final ConfigurableOpenSearchProvider provider; + + /** + * Private constructor to prevent direct instantiation + * Creates immutable provider with default configuration from properties + */ + public OpenSearchDefaultClientProvider() { + this.provider = new ConfigurableOpenSearchProvider(); + Logger.info(this.getClass(), "OpenSearchClients initialized with default configuration"); + } + + /** + * Get the OpenSearch client using default configuration from properties + * Thread-safe access to immutable provider + */ + public OpenSearchClient getClient() { + return provider.getClient(); + } + + /** + * Close resources when shutting down + */ + public void shutdown() { + Logger.info(this.getClass(), "Shutting down OpenSearch clients"); + try { + provider.close(); + } catch (IOException e) { + Logger.warn(this.getClass(), "Error closing OpenSearch provider: " + e.getMessage(), e); + } + } + + /** + * Create a default test configuration for local OpenSearch + * This is a convenience method for tests + */ + public static OpenSearchClientConfig createLocalTestConfig() { + return OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9201") // Local OpenSearch port + .tlsEnabled(false) // Security disabled + .trustSelfSigned(true) + .connectionTimeout(java.time.Duration.ofSeconds(10)) + .socketTimeout(java.time.Duration.ofSeconds(10)) + .maxConnections(50) + .maxConnectionsPerRoute(25) + .build(); + } + + /** + * Create a default production-like configuration + * This is a convenience method for different test scenarios + */ + public static OpenSearchClientConfig createProductionTestConfig() { + return OpenSearchClientConfig.builder() + .addEndpoints("https://opensearch.prod.com:9200") + .username("admin") + .password("secure_password") + .tlsEnabled(true) + .trustSelfSigned(false) + .connectionTimeout(java.time.Duration.ofSeconds(15)) + .socketTimeout(java.time.Duration.ofSeconds(30)) + .maxConnections(100) + .maxConnectionsPerRoute(50) + .build(); + } + + /** + * Create a cluster configuration for testing + * This is a convenience method for cluster test scenarios + */ + public static OpenSearchClientConfig createClusterTestConfig(String... endpoints) { + ImmutableOpenSearchClientConfig.Builder builder = OpenSearchClientConfig.builder() + .tlsEnabled(false) + .trustSelfSigned(true) + .connectionTimeout(java.time.Duration.ofSeconds(15)) + .socketTimeout(java.time.Duration.ofSeconds(30)) + .maxConnections(100) + .maxConnectionsPerRoute(30); + + for (String endpoint : endpoints) { + builder.addEndpoints(endpoint); + } + + return builder.build(); + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/contenttype/business/FieldAPIImpl.java b/dotCMS/src/main/java/com/dotcms/contenttype/business/FieldAPIImpl.java index f23ea8bed701..a802f5a6599a 100644 --- a/dotCMS/src/main/java/com/dotcms/contenttype/business/FieldAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/contenttype/business/FieldAPIImpl.java @@ -8,7 +8,7 @@ import com.dotcms.business.CloseDBIfOpened; import com.dotcms.business.WrapInTransaction; import com.dotcms.cdi.CDIUtils; -import com.dotcms.content.elasticsearch.business.IndiciesInfo; +import com.dotcms.content.elasticsearch.business.IndicesInfo; import com.dotcms.content.elasticsearch.util.ESMappingUtilHelper; import com.dotcms.contenttype.business.uniquefields.UniqueFieldValidationStrategyResolver; import com.dotcms.contenttype.exception.NotFoundInDbException; @@ -409,22 +409,22 @@ private void reorderFields(final String contentTypeId, final Field field, final */ private void addESMappingForField(final Structure structure, final Field field) { try { - final IndiciesInfo indiciesInfo = APILocator.getIndiciesAPI().loadIndicies(); - if (indiciesInfo != null){ - if (UtilMethods.isSet(indiciesInfo.getLive())) { - ESMappingUtilHelper.getInstance().addCustomMapping(field, indiciesInfo.getLive()); + final IndicesInfo legacyIndicesInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); + if (legacyIndicesInfo != null){ + if (UtilMethods.isSet(legacyIndicesInfo.getLive())) { + ESMappingUtilHelper.getInstance().addCustomMapping(field, legacyIndicesInfo.getLive()); Logger.debug(this.getClass(), () -> String.format( "Elasticsearch mapping set for Field: %s. Content type: %s on Index: %s", field.name(), structure.getName(), APILocator.getESIndexAPI() - .removeClusterIdFromName(indiciesInfo.getLive()))); + .removeClusterIdFromName(legacyIndicesInfo.getLive()))); } - if (UtilMethods.isSet(indiciesInfo.getWorking())) { - ESMappingUtilHelper.getInstance().addCustomMapping(field, indiciesInfo.getWorking()); + if (UtilMethods.isSet(legacyIndicesInfo.getWorking())) { + ESMappingUtilHelper.getInstance().addCustomMapping(field, legacyIndicesInfo.getWorking()); Logger.debug(this.getClass(), () -> String.format( "Elasticsearch mapping set for Field: %s. Content type: %s on Index: %s", field.name(), structure.getName(), APILocator.getESIndexAPI() - .removeClusterIdFromName(indiciesInfo.getWorking()))); + .removeClusterIdFromName(legacyIndicesInfo.getWorking()))); } } diff --git a/dotCMS/src/main/java/com/dotcms/featureflag/FeatureFlagName.java b/dotCMS/src/main/java/com/dotcms/featureflag/FeatureFlagName.java index 8d3c1760500c..c8a5a8612666 100644 --- a/dotCMS/src/main/java/com/dotcms/featureflag/FeatureFlagName.java +++ b/dotCMS/src/main/java/com/dotcms/featureflag/FeatureFlagName.java @@ -46,4 +46,9 @@ public interface FeatureFlagName { String FEATURE_FLAG_UVE_TOGGLE_LOCK = "FEATURE_FLAG_UVE_TOGGLE_LOCK"; String FEATURE_FLAG_UVE_STYLE_EDITOR = "FEATURE_FLAG_UVE_STYLE_EDITOR"; + + // Open Search Related + String FEATURE_FLAG_OPEN_SEARCH_WRITE = "FEATURE_FLAG_OPEN_SEARCH_WRITE"; + + String FEATURE_FLAG_OPEN_SEARCH_READ = "FEATURE_FLAG_OPEN_SEARCH_READ"; } diff --git a/dotCMS/src/main/java/com/dotcms/publishing/job/SiteSearchJobImpl.java b/dotCMS/src/main/java/com/dotcms/publishing/job/SiteSearchJobImpl.java index 4049a638eacd..68069763e1ca 100644 --- a/dotCMS/src/main/java/com/dotcms/publishing/job/SiteSearchJobImpl.java +++ b/dotCMS/src/main/java/com/dotcms/publishing/job/SiteSearchJobImpl.java @@ -2,7 +2,7 @@ import com.dotcms.content.elasticsearch.business.ESIndexAPI; import com.dotcms.content.elasticsearch.business.ESMappingAPIImpl; -import com.dotcms.content.elasticsearch.business.IndiciesAPI; +import com.dotcms.content.elasticsearch.business.IndicesAPI; import com.dotcms.enterprise.LicenseUtil; import com.dotcms.enterprise.license.LicenseLevel; import com.dotcms.enterprise.publishing.bundlers.FileAssetBundler; @@ -14,7 +14,6 @@ import com.dotcms.publishing.DotPublishingException; import com.dotcms.publishing.PublishStatus; import com.dotcms.publishing.PublisherAPI; -import com.dotcms.publishing.output.DirectoryBundleOutput; import com.dotmarketing.beans.Host; import com.dotmarketing.business.APILocator; import com.dotmarketing.business.UserAPI; @@ -80,7 +79,7 @@ public class SiteSearchJobImpl { static final String PATHS = "paths"; private final ESIndexAPI esIndexAPI; - private final IndiciesAPI indicesAPI; + private final IndicesAPI indicesAPI; private final SiteSearchAPI siteSearchAPI; private final HostAPI hostAPI; private final UserAPI userAPI; @@ -92,7 +91,7 @@ public class SiteSearchJobImpl { @VisibleForTesting SiteSearchJobImpl( final ESIndexAPI esIndexAPI, - final IndiciesAPI indicesAPI, + final IndicesAPI indicesAPI, final SiteSearchAPI siteSearchAPI, final HostAPI hostAPI, final UserAPI userAPI, diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/index/ESIndexResource.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/index/ESIndexResource.java index a1d6731d8f9a..da493998001e 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/api/v1/index/ESIndexResource.java +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/index/ESIndexResource.java @@ -34,7 +34,7 @@ import com.dotcms.content.elasticsearch.business.ContentletIndexAPIImpl; import com.dotcms.content.elasticsearch.business.ESIndexAPI; import com.dotcms.content.elasticsearch.business.ESIndexHelper; -import com.dotcms.content.elasticsearch.business.IndiciesAPI; +import com.dotcms.content.elasticsearch.business.IndicesAPI; import com.dotcms.content.elasticsearch.business.NodeStats; import com.dotcms.content.elasticsearch.util.ESReindexationProcessStatus; import com.dotcms.contenttype.model.type.ContentType; @@ -110,7 +110,7 @@ public ESIndexResource(){ @VisibleForTesting ESIndexResource(ESIndexAPI indexAPI, ESIndexHelper indexHelper, - WebResource webResource, LayoutAPI layoutAPI, IndiciesAPI indiciesAPI) { + WebResource webResource, LayoutAPI layoutAPI, IndicesAPI indicesAPI) { this.indexAPI = indexAPI; this.indexHelper = indexHelper; this.idxApi = APILocator.getContentletIndexAPI(); @@ -276,7 +276,7 @@ public Response createIndex(@Context HttpServletRequest httpServletRequest, @Con indexName=ContentletIndexAPIImpl.timestampFormatter.format(new Date()); indexName = (live) ? "live_" + indexName : "working_" + indexName; - APILocator.getContentletIndexAPI().createContentIndex(indexName, shards); + APILocator.getContentletIndexAPI().createContentIndexLegacy(indexName, shards); ESMappingUtilHelper.getInstance().addCustomMapping(indexName); return Response.ok(indexName).build(); diff --git a/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java b/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java index 0b145791cecc..0bf29f1c1fa7 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java @@ -32,8 +32,9 @@ import com.dotcms.content.elasticsearch.business.ContentletIndexAPIImpl; import com.dotcms.content.elasticsearch.business.ESContentletAPIImpl; import com.dotcms.content.elasticsearch.business.ESIndexAPI; -import com.dotcms.content.elasticsearch.business.IndiciesAPI; -import com.dotcms.content.elasticsearch.business.IndiciesAPIImpl; +import com.dotcms.content.elasticsearch.business.IndicesAPI; +import com.dotcms.content.elasticsearch.business.IndicesAPIImpl; +import com.dotcms.content.opensearch.business.OpenSearchIndexAPI; import com.dotcms.contenttype.business.ContentTypeAPI; import com.dotcms.contenttype.business.ContentTypeAPIImpl; import com.dotcms.contenttype.business.ContentTypeDestroyAPI; @@ -707,12 +708,12 @@ public static LoginServiceAPI getLoginServiceAPI(){ } /** - * Creates a single instance of the {@link IndiciesAPI} class. + * Creates a single instance of the {@link IndicesAPI} class. * - * @return The {@link IndiciesAPI} class. + * @return The {@link IndicesAPI} class. */ - public static IndiciesAPI getIndiciesAPI() { - return (IndiciesAPI) getInstance(APIIndex.INDICIES_API); + public static IndicesAPI getIndiciesAPI() { + return (IndicesAPI) getInstance(APIIndex.INDICIES_API); } /** @@ -1394,8 +1395,10 @@ enum APIIndex ACHECKER_API, CONTENT_ANALYTICS_API, JOB_QUEUE_MANAGER_API, - AI_VISION_API, - ANALYTICS_CUSTOM_ATTRIBUTE_API; + AI_VISION_API, + ANALYTICS_CUSTOM_ATTRIBUTE_API, + OS_INDEX_API + ; Object create() { switch(this) { @@ -1432,7 +1435,7 @@ Object create() { case WORKFLOW_API : return new WorkflowAPIImpl(); case CACHE_PROVIDER_API : return new CacheProviderAPIImpl(); case TAG_API: return new TagAPIImpl(); - case INDICIES_API: return new IndiciesAPIImpl(); + case INDICIES_API: return new IndicesAPIImpl(); case CONTENLET_INDEX_API: return new ContentletIndexAPIImpl(); case ES_INDEX_API: return new ESIndexAPI(); case PUBLISHER_API: return new PublisherAPIImpl(); @@ -1493,6 +1496,7 @@ Object create() { case AI_VISION_API: return new OpenAIVisionAPIImpl(); case ANALYTICS_CUSTOM_ATTRIBUTE_API: return new CustomAttributeAPIImpl(); + case OS_INDEX_API: return CDIUtils.getBeanThrows(OpenSearchIndexAPI.class); } throw new AssertionError("Unknown API index: " + this); } diff --git a/dotCMS/src/main/java/com/dotmarketing/business/CacheLocator.java b/dotCMS/src/main/java/com/dotmarketing/business/CacheLocator.java index bf396580d1ee..b8412fb5ad7b 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/CacheLocator.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/CacheLocator.java @@ -7,8 +7,8 @@ import com.dotcms.cache.KeyValueCache; import com.dotcms.cache.KeyValueCacheImpl; import com.dotcms.content.elasticsearch.ESQueryCache; -import com.dotcms.content.elasticsearch.business.IndiciesCache; -import com.dotcms.content.elasticsearch.business.IndiciesCacheImpl; +import com.dotcms.content.elasticsearch.business.IndicesCache; +import com.dotcms.content.elasticsearch.business.IndicesCacheImpl; import com.dotcms.contenttype.business.ContentTypeCache2; import com.dotcms.contenttype.business.ContentTypeCache2Impl; import com.dotcms.csspreproc.CSSCache; @@ -255,8 +255,8 @@ public static HostVariablesCache getHostVariablesCache() { return (HostVariablesCache)getInstance(CacheIndex.HostVariables); } - public static IndiciesCache getIndiciesCache() { - return (IndiciesCache)getInstance(CacheIndex.Indicies); + public static IndicesCache getIndiciesCache() { + return (IndicesCache)getInstance(CacheIndex.Indicies); } @@ -529,7 +529,7 @@ Cachable create() { case Versionable : return new VersionableCacheImpl(); case FolderCache : return new FolderCacheImpl(); case WorkflowCache : return new WorkflowCacheImpl(); - case Indicies: return new IndiciesCacheImpl(); + case Indicies: return new IndicesCacheImpl(); case NavTool: return new NavToolCacheImpl(); case PublishingEndPoint: return new PublishingEndPointCacheImpl(); case PushedAssets: return new PushedAssetsCacheImpl(); diff --git a/dotCMS/src/main/java/com/dotmarketing/business/FactoryLocator.java b/dotCMS/src/main/java/com/dotmarketing/business/FactoryLocator.java index 758b0bec5c11..d5e0984f4336 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/FactoryLocator.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/FactoryLocator.java @@ -8,7 +8,7 @@ import com.dotcms.cdi.CDIUtils; import com.dotcms.cluster.business.ServerFactory; import com.dotcms.content.elasticsearch.business.ESContentFactoryImpl; -import com.dotcms.content.elasticsearch.business.IndiciesFactory; +import com.dotcms.content.elasticsearch.business.IndicesFactory; import com.dotcms.contenttype.business.ContentTypeFactory; import com.dotcms.contenttype.business.ContentTypeFactoryImpl; import com.dotcms.contenttype.business.FieldFactory; @@ -16,7 +16,6 @@ import com.dotcms.contenttype.business.RelationshipFactory; import com.dotcms.contenttype.business.RelationshipFactoryImpl; import com.dotcms.cube.CubeJSClientFactory; -import com.dotcms.cube.CubeJSClientFactoryImpl; import com.dotcms.enterprise.DashboardProxy; import com.dotcms.enterprise.RulesFactoryProxy; import com.dotcms.enterprise.ServerActionFactoryImplProxy; @@ -186,8 +185,8 @@ public static FolderFactory getFolderFactory(){ public static WorkFlowFactory getWorkFlowFactory(){ return (WorkFlowFactory) getInstance(FactoryIndex.WORKFLOWS_FACTORY); } - public static IndiciesFactory getIndiciesFactory(){ - return (IndiciesFactory) getInstance(FactoryIndex.INDICIES_FACTORY); + public static IndicesFactory getIndiciesFactory(){ + return (IndicesFactory) getInstance(FactoryIndex.INDICIES_FACTORY); } public static LinkCheckerFactory getLinkCheckerFactory() { @@ -410,7 +409,7 @@ Object create() { case VERSIONABLE_FACTORY : return new VersionableFactoryImpl(); case FOLDER_FACTORY : return new FolderFactoryImpl(); case WORKFLOWS_FACTORY :return new WorkflowFactoryImpl(); - case INDICIES_FACTORY: return new IndiciesFactory(); + case INDICIES_FACTORY: return new IndicesFactory(); case LINKCHECKER_FACTORY: return new LinkCheckerFactoryImpl(); case PUBLISHER_END_POINT_FACTORY: return new PublishingEndPointFactoryImpl(); case ENVIRONMENT_FACTORY: return new EnvironmentFactoryImpl(); diff --git a/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java index 18729de8e963..7dd04728e794 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java @@ -2537,7 +2537,7 @@ void resetChildrenPermissionReferences(Structure structure) throws DotDataExcept throw new RuntimeException(e); } - BulkRequest bulkRequest=indexAPI.createBulkRequest(); + BulkRequest bulkRequest=indexAPI.createBulkRequestLegacy(); bulkRequest.timeout(TimeValue.timeValueMillis(INDEX_OPERATIONS_TIMEOUT_IN_MS)); for(Contentlet cont : contentlets) { diff --git a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReIndexerStatus.java b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReIndexerStatus.java index a238495b405d..3b5166965732 100644 --- a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReIndexerStatus.java +++ b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReIndexerStatus.java @@ -7,7 +7,7 @@ import com.dotcms.business.CloseDBIfOpened; import com.dotcms.content.elasticsearch.business.ContentletIndexAPIImpl; -import com.dotcms.content.elasticsearch.business.IndiciesInfo; +import com.dotcms.content.elasticsearch.business.IndicesInfo; import com.dotmarketing.business.APILocator; import com.dotmarketing.common.db.DotConnect; import com.dotmarketing.exception.DotDataException; @@ -62,14 +62,14 @@ public int getLastIndexationProgress(int countToIndex) throws DotDataException { @CloseDBIfOpened public String currentIndexPath() throws DotDataException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); return "[" + esIndexAPI.removeClusterIdFromName(info.getWorking()) + "," + esIndexAPI.removeClusterIdFromName(info.getLive()) + "]"; } @CloseDBIfOpened public String getNewIndexPath() throws DotDataException { - final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo info = APILocator.getIndiciesAPI().loadLegacyIndices(); return "[" + esIndexAPI.removeClusterIdFromName(info.getReindexWorking()) + "," + esIndexAPI.removeClusterIdFromName(info.getReindexLive()) + "]"; } diff --git a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexThread.java b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexThread.java index 9f2433265b6c..076af0fc994a 100644 --- a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexThread.java +++ b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexThread.java @@ -239,10 +239,10 @@ private void runReindexLoop() { if (bulkProcessor == null || rebuildBulkIndexer.get()) { closeBulkProcessor(bulkProcessor); bulkProcessorListener = new BulkProcessorListener(); - bulkProcessor = indexAPI.createBulkProcessor(bulkProcessorListener); + bulkProcessor = indexAPI.createBulkProcessorLegacy(bulkProcessorListener); } bulkProcessorListener.workingRecords.putAll(workingRecords); - indexAPI.appendToBulkProcessor(bulkProcessor, workingRecords.values()); + indexAPI.appendToBulkProcessorLegacy(bulkProcessor, workingRecords.values()); contentletsIndexed += bulkProcessorListener.getContentletsIndexed(); // otherwise, reindex normally diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/ajax/IndexAjaxAction.java b/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/ajax/IndexAjaxAction.java index f3a4ebd7b24b..abc4a2299ad3 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/ajax/IndexAjaxAction.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/ajax/IndexAjaxAction.java @@ -125,7 +125,7 @@ public void createIndex(HttpServletRequest request, HttpServletResponse response final boolean live = map.get("live") != null; final String indexName=((live) ? "live_" : "working_" ) + APILocator.getContentletIndexAPI().timestampFormatter.format(new Date()); - APILocator.getContentletIndexAPI().createContentIndex(indexName, shards); + APILocator.getContentletIndexAPI().createContentIndexLegacy(indexName, shards); ESMappingUtilHelper.getInstance().addCustomMapping(indexName); } diff --git a/dotCMS/src/main/java/com/dotmarketing/sitesearch/ajax/SiteSearchAjaxAction.java b/dotCMS/src/main/java/com/dotmarketing/sitesearch/ajax/SiteSearchAjaxAction.java index a06f426391fc..6d3d7696b62d 100644 --- a/dotCMS/src/main/java/com/dotmarketing/sitesearch/ajax/SiteSearchAjaxAction.java +++ b/dotCMS/src/main/java/com/dotmarketing/sitesearch/ajax/SiteSearchAjaxAction.java @@ -265,7 +265,7 @@ public void getIndexStatus(HttpServletRequest request, HttpServletResponse respo String indexName = ESIndexHelper.getInstance().getIndexNameOrAlias(map,"indexName", "indexAlias", APILocator.getESIndexAPI()); response.setContentType("text/plain"); - response.getWriter().println(APILocator.getIndiciesAPI().loadIndicies().getSiteSearch().equals(indexName) ? "default" : "inactive"); + response.getWriter().println(APILocator.getIndiciesAPI().loadLegacyIndices().getSiteSearch().equals(indexName) ? "default" : "inactive"); } catch(Exception ex) { throw new RuntimeException(ex); @@ -275,7 +275,7 @@ public void getIndexStatus(HttpServletRequest request, HttpServletResponse respo @Override public void getNotActiveIndexNames(HttpServletRequest request, HttpServletResponse response) throws IOException { try { - String defaultIndex=APILocator.getIndiciesAPI().loadIndicies().getSiteSearch(); + String defaultIndex=APILocator.getIndiciesAPI().loadLegacyIndices().getSiteSearch(); List ret=new ArrayList<>(); for(String ii : APILocator.getSiteSearchAPI().listIndices()) if(defaultIndex==null || !defaultIndex.equals(ii)) diff --git a/dotCMS/src/main/java/com/dotmarketing/startup/runonce/Task251212AddVersionColumnIndicesTable.java b/dotCMS/src/main/java/com/dotmarketing/startup/runonce/Task251212AddVersionColumnIndicesTable.java new file mode 100644 index 000000000000..b50c43693906 --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/startup/runonce/Task251212AddVersionColumnIndicesTable.java @@ -0,0 +1,44 @@ +package com.dotmarketing.startup.runonce; + +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.common.db.DotDatabaseMetaData; +import com.dotmarketing.db.DbConnectionFactory; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.startup.StartupTask; +import com.dotmarketing.util.Logger; +import java.sql.SQLException; + +public class Task251212AddVersionColumnIndicesTable implements StartupTask { + + @Override + public boolean forceRun() { + try { + return !new DotDatabaseMetaData().hasColumn("indicies", "index_version"); + } catch (SQLException e) { + Logger.error(this, e.getMessage(),e); + return false; + } + } + + @Override + public void executeUpgrade() throws DotDataException, DotRuntimeException { + try { + DbConnectionFactory.getConnection().setAutoCommit(true); + } catch (SQLException e) { + throw new DotDataException(e.getMessage(), e); + } + final DotConnect dropIndexStatement = new DotConnect().setSQL("ALTER TABLE indicies DROP CONSTRAINT IF EXISTS indicies_index_type_key"); + dropIndexStatement.loadResult(); + + final DotConnect addColumnStatement = new DotConnect().setSQL("ALTER TABLE indicies ADD COLUMN IF NOT EXISTS index_version varchar(16) NULL"); + addColumnStatement.loadResult(); + + //Only one type (working,live,site-search) of index per version can exist + final DotConnect addIndexStatement = new DotConnect().setSQL("ALTER TABLE indicies ADD CONSTRAINT uq_index_type_version UNIQUE (index_type, index_version)"); + addIndexStatement.loadResult(); + + } + + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java b/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java index 7ab3796a3582..0118ede48683 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java @@ -259,6 +259,7 @@ import com.dotmarketing.startup.runonce.Task250828CreateCustomAttributeTable; import com.dotmarketing.startup.runonce.Task251029RemoveContentTypesLegacyPortletFromLayouts; import com.dotmarketing.startup.runonce.Task251103AddStylePropertiesColumnInMultiTree; +import com.dotmarketing.startup.runonce.Task251212AddVersionColumnIndicesTable; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -592,6 +593,7 @@ public static List> getStartupRunOnceTaskClasses() { .add(Task250828CreateCustomAttributeTable.class) .add(Task251029RemoveContentTypesLegacyPortletFromLayouts.class) .add(Task251103AddStylePropertiesColumnInMultiTree.class) + .add(Task251212AddVersionColumnIndicesTable.class) .build(); return ret.stream().sorted(classNameComparator).collect(Collectors.toList()); diff --git a/dotCMS/src/main/resources/postgres.sql b/dotCMS/src/main/resources/postgres.sql index b71ecd19341f..c04f25e20a3f 100644 --- a/dotCMS/src/main/resources/postgres.sql +++ b/dotCMS/src/main/resources/postgres.sql @@ -2203,8 +2203,14 @@ ALTER TABLE tag ALTER COLUMN user_id TYPE text; -- ****** Indicies Data Storage ******* create table indicies ( index_name varchar(100) primary key, - index_type varchar(16) not null unique + index_type varchar(16) not null, + index_version varchar(16) null ); +-- We can only have one index type per version +ALTER TABLE indicies + ADD CONSTRAINT uq_index_type_version + UNIQUE (index_type, index_version); + -- ****** Log Console Table ******* CREATE TABLE log_mapper ( enabled numeric(1,0) NOT null, diff --git a/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_index_stats.jsp b/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_index_stats.jsp index b6f82913f6a9..7aa9e84698dc 100644 --- a/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_index_stats.jsp +++ b/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_index_stats.jsp @@ -1,6 +1,6 @@ <%@page import="com.dotcms.cluster.ClusterUtils"%> <%@page import="com.dotcms.content.elasticsearch.business.ESIndexAPI"%> -<%@page import="com.dotcms.content.elasticsearch.business.IndiciesInfo"%> +<%@page import="com.dotcms.content.elasticsearch.business.IndicesInfo"%> <%@page import="com.dotmarketing.business.APILocator"%> <%@page import="com.dotmarketing.exception.DotSecurityException"%> <%@page import="com.dotmarketing.portlets.structure.factories.StructureFactory"%> @@ -17,7 +17,7 @@ SiteSearchAPI ssapi = APILocator.getSiteSearchAPI(); ESIndexAPI esapi = APILocator.getESIndexAPI(); -IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies(); +IndicesInfo info=APILocator.getIndiciesAPI().loadLegacyIndices(); try { user = com.liferay.portal.util.PortalUtil.getUser(request); diff --git a/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_job_schedule.jsp b/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_job_schedule.jsp index 9bcdda5cdf52..82d28a0972aa 100644 --- a/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_job_schedule.jsp +++ b/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/site_search_job_schedule.jsp @@ -66,7 +66,7 @@ for(String x : indexHosts){ catch(Exception e){} } -boolean hasDefaultIndex = APILocator.getIndiciesAPI().loadIndicies().getSiteSearch() != null; +boolean hasDefaultIndex = APILocator.getIndiciesAPI().loadLegacyIndices().getSiteSearch() != null; List langs=APILocator.getLanguageAPI().getLanguages(); @@ -77,7 +77,7 @@ String includeExclude = (String) props.get("includeExclude") ==null ? "all": (St boolean hasPath = false; -final String siteSearch = APILocator.getIndiciesAPI().loadIndicies().getSiteSearch(); +final String siteSearch = APILocator.getIndiciesAPI().loadLegacyIndices().getSiteSearch(); %> diff --git a/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/test_site_search.jsp b/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/test_site_search.jsp index 3b06576537ee..9badddf1c8fe 100644 --- a/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/test_site_search.jsp +++ b/dotCMS/src/main/webapp/html/portlet/ext/sitesearch/test_site_search.jsp @@ -1,5 +1,4 @@ <%@page import="com.dotcms.content.elasticsearch.business.ESIndexAPI"%> -<%@page import="com.dotcms.content.elasticsearch.business.IndiciesInfo"%> <%@page import="com.dotcms.enterprise.publishing.sitesearch.SiteSearchResults"%> <%@page import="com.dotmarketing.business.APILocator"%> <%@page import="com.dotmarketing.exception.DotSecurityException"%> @@ -9,13 +8,14 @@ <%@page import="com.dotmarketing.util.Logger"%> <%@page import="org.elasticsearch.cluster.health.ClusterIndexHealth"%> <%@ page import="com.dotcms.content.elasticsearch.business.IndexStats" %> +<%@ page import="com.dotcms.content.elasticsearch.business.IndicesInfo" %> <%@ include file="/html/common/init.jsp"%> <% List structs = StructureFactory.getStructures(); SiteSearchAPI ssapi = APILocator.getSiteSearchAPI(); ESIndexAPI esapi = APILocator.getESIndexAPI(); -IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies(); +IndicesInfo info=APILocator.getIndiciesAPI().loadLegacyIndices(); @@ -108,7 +108,7 @@ dojo.connect(dijit.byId("testQuery"), 'onkeypress', function (evt) { diff --git a/dotCMS/src/test/java/com/dotcms/content/opensearch/util/OpenSearchClientConfigTest.java b/dotCMS/src/test/java/com/dotcms/content/opensearch/util/OpenSearchClientConfigTest.java new file mode 100644 index 000000000000..2e9d8953edbf --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/content/opensearch/util/OpenSearchClientConfigTest.java @@ -0,0 +1,332 @@ +package com.dotcms.content.opensearch.util; + +import org.junit.Test; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.*; + +/** + * Unit test for OpenSearchClientConfig. + * Tests the configuration builder pattern and validation logic. + * + * @author fabrizio + */ +public class OpenSearchClientConfigTest { + + /** + * Test basic configuration creation with minimal settings + */ + @Test + public void test_createConfig_withMinimalSettings_shouldSucceed() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9200") + .build(); + + // Assert + assertNotNull("Config should not be null", config); + assertEquals("Should have one endpoint", 1, config.endpoints().size()); + assertEquals("Endpoint should match", "http://localhost:9200", config.endpoints().get(0)); + assertFalse("Username should be empty", config.username().isPresent()); + assertFalse("Password should be empty", config.password().isPresent()); + assertFalse("JWT token should be empty", config.jwtToken().isPresent()); + assertFalse("TLS should be disabled by default", config.tlsEnabled()); + assertFalse("Trust self-signed should be false by default", config.trustSelfSigned()); + } + + /** + * Test configuration creation with all settings + */ + @Test + public void test_createConfig_withAllSettings_shouldSucceed() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .endpoints(Arrays.asList("https://node1:9200", "https://node2:9200")) + .username("admin") + .password("password123") + .clientCertPath("/path/to/cert.pem") + .clientKeyPath("/path/to/key.pem") + .caCertPath("/path/to/ca.pem") + .tlsEnabled(true) + .trustSelfSigned(false) + .connectionTimeout(Duration.ofSeconds(15)) + .socketTimeout(Duration.ofSeconds(45)) + .maxConnections(200) + .maxConnectionsPerRoute(100) + .build(); + + // Assert + assertNotNull("Config should not be null", config); + assertEquals("Should have two endpoints", 2, config.endpoints().size()); + assertTrue("Should have first endpoint", config.endpoints().contains("https://node1:9200")); + assertTrue("Should have second endpoint", config.endpoints().contains("https://node2:9200")); + assertTrue("Username should be present", config.username().isPresent()); + assertEquals("Username should match", "admin", config.username().get()); + assertTrue("Password should be present", config.password().isPresent()); + assertEquals("Password should match", "password123", config.password().get()); + assertTrue("Client cert path should be present", config.clientCertPath().isPresent()); + assertEquals("Client cert path should match", "/path/to/cert.pem", config.clientCertPath().get()); + assertTrue("TLS should be enabled", config.tlsEnabled()); + assertFalse("Trust self-signed should be false", config.trustSelfSigned()); + assertEquals("Connection timeout should match", Duration.ofSeconds(15), config.connectionTimeout()); + assertEquals("Socket timeout should match", Duration.ofSeconds(45), config.socketTimeout()); + assertEquals("Max connections should match", 200, config.maxConnections()); + assertEquals("Max connections per route should match", 100, config.maxConnectionsPerRoute()); + } + + /** + * Test configuration with basic authentication + */ + @Test + public void test_createConfig_withBasicAuth_shouldSucceed() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .username("user") + .password("pass") + .tlsEnabled(true) + .build(); + + // Assert + assertTrue("Username should be present", config.username().isPresent()); + assertTrue("Password should be present", config.password().isPresent()); + assertEquals("Username should match", "user", config.username().get()); + assertEquals("Password should match", "pass", config.password().get()); + assertFalse("JWT token should not be present", config.jwtToken().isPresent()); + assertFalse("Client cert should not be present", config.clientCertPath().isPresent()); + } + + /** + * Test configuration with JWT authentication + */ + @Test + public void test_createConfig_withJWTAuth_shouldSucceed() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .jwtToken("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.token") + .tlsEnabled(true) + .build(); + + // Assert + assertTrue("JWT token should be present", config.jwtToken().isPresent()); + assertEquals("JWT token should match", "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.token", config.jwtToken().get()); + assertFalse("Username should not be present", config.username().isPresent()); + assertFalse("Password should not be present", config.password().isPresent()); + assertFalse("Client cert should not be present", config.clientCertPath().isPresent()); + } + + /** + * Test configuration with certificate authentication + */ + @Test + public void test_createConfig_withCertAuth_shouldSucceed() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .clientCertPath("/path/to/client.pem") + .clientKeyPath("/path/to/client.key") + .caCertPath("/path/to/ca.pem") + .tlsEnabled(true) + .build(); + + // Assert + assertTrue("Client cert path should be present", config.clientCertPath().isPresent()); + assertTrue("Client key path should be present", config.clientKeyPath().isPresent()); + assertTrue("CA cert path should be present", config.caCertPath().isPresent()); + assertEquals("Client cert path should match", "/path/to/client.pem", config.clientCertPath().get()); + assertEquals("Client key path should match", "/path/to/client.key", config.clientKeyPath().get()); + assertEquals("CA cert path should match", "/path/to/ca.pem", config.caCertPath().get()); + assertFalse("Username should not be present", config.username().isPresent()); + assertFalse("JWT token should not be present", config.jwtToken().isPresent()); + } + + /** + * Test default values + */ + @Test + public void test_createConfig_withDefaults_shouldHaveCorrectDefaults() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9200") + .build(); + + // Assert - Test default values + assertFalse("TLS should be disabled by default", config.tlsEnabled()); + assertFalse("Trust self-signed should be false by default", config.trustSelfSigned()); + assertEquals("Default connection timeout should be 10 seconds", + Duration.ofSeconds(10), config.connectionTimeout()); + assertEquals("Default socket timeout should be 30 seconds", + Duration.ofSeconds(30), config.socketTimeout()); + assertEquals("Default max connections should be 100", 100, config.maxConnections()); + assertEquals("Default max connections per route should be 50", 50, config.maxConnectionsPerRoute()); + } + + /** + * Test validation - empty endpoints list should fail + */ + @Test(expected = IllegalArgumentException.class) + public void test_createConfig_withEmptyEndpoints_shouldThrowException() { + // Act & Assert + OpenSearchClientConfig.builder() + .endpoints(Collections.emptyList()) + .build(); + } + + /** + * Test validation - null endpoint should fail + */ + @Test(expected = IllegalArgumentException.class) + public void test_createConfig_withNullEndpoint_shouldThrowException() { + // Act & Assert + OpenSearchClientConfig.builder() + .endpoints(Arrays.asList("http://localhost:9200", null)) + .build(); + } + + /** + * Test validation - empty endpoint should fail + */ + @Test(expected = IllegalArgumentException.class) + public void test_createConfig_withEmptyEndpoint_shouldThrowException() { + // Act & Assert + OpenSearchClientConfig.builder() + .endpoints(Arrays.asList("http://localhost:9200", "")) + .build(); + } + + /** + * Test validation - conflicting authentication methods (basic + JWT) should fail + */ + @Test(expected = IllegalArgumentException.class) + public void test_createConfig_withBasicAndJWTAuth_shouldThrowException() { + // Act & Assert + OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .username("user") + .password("pass") + .jwtToken("token") + .build(); + } + + /** + * Test validation - conflicting authentication methods (basic + cert) should fail + */ + @Test(expected = IllegalArgumentException.class) + public void test_createConfig_withBasicAndCertAuth_shouldThrowException() { + // Act & Assert + OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .username("user") + .password("pass") + .clientCertPath("/path/to/cert.pem") + .build(); + } + + /** + * Test validation - conflicting authentication methods (JWT + cert) should fail + */ + @Test(expected = IllegalArgumentException.class) + public void test_createConfig_withJWTAndCertAuth_shouldThrowException() { + // Act & Assert + OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .jwtToken("token") + .clientCertPath("/path/to/cert.pem") + .clientKeyPath("/path/to/key.pem") + .build(); + } + + /** + * Test validation - incomplete basic auth (only username) should pass + * Note: This is allowed as password could be empty for some auth scenarios + */ + @Test + public void test_createConfig_withOnlyUsername_shouldSucceed() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .username("user") + .build(); + + // Assert + assertTrue("Username should be present", config.username().isPresent()); + assertFalse("Password should not be present", config.password().isPresent()); + } + + /** + * Test validation - incomplete cert auth (only client cert) should pass + * Note: This allows partial certificate configuration + */ + @Test + public void test_createConfig_withOnlyClientCert_shouldSucceed() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("https://localhost:9200") + .clientCertPath("/path/to/cert.pem") + .build(); + + // Assert + assertTrue("Client cert should be present", config.clientCertPath().isPresent()); + assertFalse("Client key should not be present", config.clientKeyPath().isPresent()); + } + + /** + * Test immutability - config should be immutable after creation + */ + @Test + public void test_config_shouldBeImmutable() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9200") + .build(); + + // Assert - The endpoints list should be immutable + try { + config.endpoints().add("http://another:9200"); + fail("Should not be able to modify endpoints list"); + } catch (UnsupportedOperationException e) { + // Expected - list should be immutable + } + } + + /** + * Test builder reuse - builder should be reusable + */ + @Test + public void test_builderReuse_shouldCreateDifferentConfigs() { + // Act + ImmutableOpenSearchClientConfig.Builder builder = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9200"); + + OpenSearchClientConfig config1 = builder.username("user1").build(); + OpenSearchClientConfig config2 = builder.username("user2").build(); + + // Assert + assertEquals("First config should have user1", "user1", config1.username().get()); + assertEquals("Second config should have user2", "user2", config2.username().get()); + } + + /** + * Test multiple endpoints addition + */ + @Test + public void test_addMultipleEndpoints_shouldAcceptAllEndpoints() { + // Act + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("http://node1:9200") + .addEndpoints("http://node2:9200", "http://node3:9200") + .addEndpoints("http://node4:9200") + .build(); + + // Assert + assertEquals("Should have 4 endpoints", 4, config.endpoints().size()); + assertTrue("Should contain node1", config.endpoints().contains("http://node1:9200")); + assertTrue("Should contain node2", config.endpoints().contains("http://node2:9200")); + assertTrue("Should contain node3", config.endpoints().contains("http://node3:9200")); + assertTrue("Should contain node4", config.endpoints().contains("http://node4:9200")); + } +} \ No newline at end of file diff --git a/dotcms-integration/pom.xml b/dotcms-integration/pom.xml index a29ba448ea36..a44b109f268a 100644 --- a/dotcms-integration/pom.xml +++ b/dotcms-integration/pom.xml @@ -241,6 +241,41 @@ weld-se-shaded test + + + org.opensearch.client + opensearch-java + + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5 + + + test + + + + org.apache.httpcomponents.core5 + httpcore5 + test + + + + org.apache.httpcomponents.core5 + httpcore5-h2 + test + + + + org.apache.httpcomponents.client5 + httpclient5 + test + + diff --git a/dotcms-integration/src/test/java/com/dotcms/MainSuite1b.java b/dotcms-integration/src/test/java/com/dotcms/MainSuite1b.java index 5e2c346a8c3e..c18e52f9abd4 100644 --- a/dotcms-integration/src/test/java/com/dotcms/MainSuite1b.java +++ b/dotcms-integration/src/test/java/com/dotcms/MainSuite1b.java @@ -1,5 +1,6 @@ package com.dotcms; +import com.dotcms.content.elasticsearch.business.IndicesFactoryTest; import com.dotcms.graphql.DotGraphQLHttpServletTest; import com.dotcms.junit.MainBaseSuite; import com.dotcms.storage.Chainable404StorageCacheTest; @@ -83,7 +84,7 @@ DotConnectTest.class, com.dotcms.contenttype.model.field.layout.FieldUtilTest.class, com.dotmarketing.portlets.contentlet.business.HostAPITest.class, - com.dotcms.content.elasticsearch.business.IndiciesFactoryTest.class, + IndicesFactoryTest.class, com.dotcms.content.elasticsearch.business.ESIndexSpeedTest.class, com.dotcms.content.elasticsearch.business.ESSiteSearchAPITest.class, com.dotcms.content.elasticsearch.business.ContentletIndexAPIImplTest.class, diff --git a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java index e593be8e7411..3329511ae4fb 100644 --- a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java @@ -45,7 +45,6 @@ import com.dotmarketing.portlets.structure.model.Structure; import com.dotmarketing.portlets.templates.model.Template; import com.dotmarketing.sitesearch.business.SiteSearchAPI; -import com.dotmarketing.util.Config; import com.dotmarketing.util.Logger; import com.dotmarketing.util.UUIDGenerator; import com.dotmarketing.util.UtilMethods; @@ -273,7 +272,7 @@ public void test_indexContentList_with_diff_refresh_strategies() throws Exceptio /** - * Testing the {@link ContentletIndexAPI#createContentIndex(String)}, + * Testing the {@link ContentletIndexAPI#createContentIndexLegacy(String)}, * {@link ContentletIndexAPI#delete(String)} and {@link ContentletIndexAPI#listDotCMSIndices()} * methods * @@ -297,12 +296,12 @@ public void createContentIndexAndDelete() throws Exception { int oldIndices = indices.size(); //Creates the working index - boolean result = indexAPI.createContentIndex(workingIndex); + boolean result = indexAPI.createContentIndexLegacy(workingIndex); //Validate assertTrue(result); //Creates the live index - result = indexAPI.createContentIndex(liveIndex); + result = indexAPI.createContentIndexLegacy(liveIndex); //Validate assertTrue(result); @@ -385,13 +384,13 @@ public void activateDeactivateIndex() throws Exception { String oldActiveWorking = indexAPI.getActiveIndexName(IndexType.WORKING.getPrefix()); //Creates the working index - boolean result = indexAPI.createContentIndex(workingIndex); + boolean result = indexAPI.createContentIndexLegacy(workingIndex); assertTrue(result); //Activate this working index indexAPI.activateIndex(workingIndex); //Creates the live index - result = indexAPI.createContentIndex(liveIndex); + result = indexAPI.createContentIndexLegacy(liveIndex); assertTrue(result); //Activate this live index indexAPI.activateIndex(liveIndex); @@ -455,12 +454,12 @@ public void optimize() throws Exception { String liveIndex = IndexType.LIVE.getPrefix() + "_" + timeStamp; //Creates the working index - boolean result = indexAPI.createContentIndex(workingIndex); + boolean result = indexAPI.createContentIndexLegacy(workingIndex); //Validate assertTrue(result); //Creates the live index - result = indexAPI.createContentIndex(liveIndex); + result = indexAPI.createContentIndexLegacy(liveIndex); //Validate assertTrue(result); @@ -1103,9 +1102,9 @@ private SearchHits indexSearch(String indexName, String query) throws Exception @Nullable private static String getSiteSearchIndex() { String indexToHit; - IndiciesInfo info; + IndicesInfo info; try { - info = APILocator.getIndiciesAPI().loadIndicies(); + info = APILocator.getIndiciesAPI().loadLegacyIndices(); } catch (DotDataException ee) { Logger.fatal(ContentletIndexAPIImpl.class, "Can't get indicies information", ee); return null; @@ -1207,9 +1206,9 @@ public void testRemoveContentFromIndex() throws DotDataException, DotSecurityExc Map entries = APILocator.getReindexQueueAPI().findContentToReindex(); - final BulkRequest bulk = indexAPI.createBulkRequest(); + final BulkRequest bulk = indexAPI.createBulkRequestLegacy(); bulk.setRefreshPolicy(RefreshPolicy.IMMEDIATE); - indexAPI.appendBulkRequest(bulk, entries.values()); + indexAPI.appendBulkRequestLegacy(bulk, entries.values()); indexAPI.putToIndex(bulk); assertEquals(0, contentletAPI diff --git a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESIndexAPITest.java b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESIndexAPITest.java index 005fb8cd4c08..10ecaaa279dc 100644 --- a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESIndexAPITest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESIndexAPITest.java @@ -1,6 +1,6 @@ package com.dotcms.content.elasticsearch.business; -import static com.dotcms.content.elasticsearch.business.IndiciesInfo.CLUSTER_PREFIX; +import static com.dotcms.content.elasticsearch.business.LegacyIndicesInfo.CLUSTER_PREFIX; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -86,7 +86,7 @@ public void index_exists_should_resolve_even_with_cluster_id() throws Exception @Test - public void testGetIndicesStatsWhenStatsTypeIsLongShouldPass() { + public void testGetIndicesLegacyStatsWhenStatsTypeIsLongShouldPass() { final Map jsonMap = new HashMap<>(); @@ -121,7 +121,7 @@ public void testGetIndicesStatsWhenStatsTypeIsLongShouldPass() { } @Test - public void testGetIndicesStatsWhenStatsTypeIsIntegerShouldPass() { + public void testGetIndicesLegacyStatsWhenStatsTypeIsIntegerShouldPass() { final Map jsonMap = new HashMap<>(); @@ -267,7 +267,7 @@ public static Object[] testDeleteOldIndicesDP() { @UseDataProvider("testDeleteOldIndicesDP") public void testDeleteOldIndices(final int inactiveLiveWorkingSetsToKeep) throws DotIndexException, IOException, InterruptedException { // get live and working active indices - final IndiciesInfo info = Try.of(()->APILocator.getIndiciesAPI().loadIndicies()) + final LegacyIndicesInfo info = Try.of(()->(LegacyIndicesInfo)APILocator.getIndiciesAPI().loadLegacyIndices()) .getOrNull(); final String liveIndex = info.getLive(); @@ -280,9 +280,9 @@ public void testDeleteOldIndices(final int inactiveLiveWorkingSetsToKeep) throws List indicesThatShouldStay = new ArrayList<>(); for(int i=0; i<6; i++) { - info.createNewIndiciesName(IndexType.REINDEX_WORKING, IndexType.REINDEX_LIVE); - contentletIndexAPI.createContentIndex(info.getReindexWorking(), 0); - contentletIndexAPI.createContentIndex(info.getReindexLive(), 0); + info.createNewIndicesName(IndexType.REINDEX_WORKING, IndexType.REINDEX_LIVE); + contentletIndexAPI.createContentIndexLegacy(info.getReindexWorking(), 0); + contentletIndexAPI.createContentIndexLegacy(info.getReindexLive(), 0); if(i>=6-inactiveLiveWorkingSetsToKeep) { indicesThatShouldStay.add(esIndexAPI.removeClusterIdFromName(info.getReindexWorking())); diff --git a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESSiteSearchAPITest.java b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESSiteSearchAPITest.java index 4bcc115ac3d8..14188f4fa358 100644 --- a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESSiteSearchAPITest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESSiteSearchAPITest.java @@ -4,7 +4,6 @@ import static org.junit.Assert.*; import com.dotcms.LicenseTestUtil; -import com.dotcms.rest.api.v1.menu.MenuResource; import com.dotcms.util.IntegrationTestInitService; import com.dotmarketing.business.APILocator; import com.dotmarketing.business.CacheLocator; @@ -12,9 +11,7 @@ import com.dotmarketing.sitesearch.business.SiteSearchAPI; import java.io.IOException; import java.util.Date; -import java.util.Set; -import com.liferay.portal.model.User; import java.util.List; import org.junit.BeforeClass; @@ -27,7 +24,7 @@ public class ESSiteSearchAPITest { private static SiteSearchAPI siteSearchAPI; private static ESIndexAPI indexAPI; - private static IndiciesAPI indiciesAPI; + private static IndicesAPI indicesAPI; private static ContentletIndexAPI contentletIndexAPI; @BeforeClass @@ -38,7 +35,7 @@ public static void prepare() throws Exception { siteSearchAPI = APILocator.getSiteSearchAPI(); indexAPI = APILocator.getESIndexAPI(); - indiciesAPI = APILocator.getIndiciesAPI(); + indicesAPI = APILocator.getIndiciesAPI(); contentletIndexAPI = APILocator.getContentletIndexAPI(); } @@ -79,14 +76,14 @@ public void testCreateSiteSearchIndexAndMakeItDefault() throws IOException, DotD assertTrue(indexAPI.listIndices().contains(indexName)); //verifies that there is no a default site search index - assertTrue(indiciesAPI.loadIndicies().getSiteSearch() == null || !indiciesAPI - .loadIndicies().getSiteSearch().equals(indexName)); + assertTrue(indicesAPI.loadLegacyIndices().getSiteSearch() == null || !indicesAPI + .loadLegacyIndices().getSiteSearch().equals(indexName)); siteSearchAPI.activateIndex(indexName); try { CacheLocator.getIndiciesCache().clearCache(); - assertNotNull(indiciesAPI.loadIndicies().getSiteSearch()); - assertTrue(indiciesAPI.loadIndicies().getSiteSearch().equals(indexName)); + assertNotNull(indicesAPI.loadLegacyIndices().getSiteSearch()); + assertTrue(indicesAPI.loadLegacyIndices().getSiteSearch().equals(indexName)); assertEquals(aliasName, indexAPI.getIndexAlias(indexName)); } finally { siteSearchAPI.deactivateIndex(indexName); @@ -112,8 +109,8 @@ public void testFullReindexKeepsDefaultSiteSearchIndex() try { indexTimestamp = contentletIndexAPI.fullReindexStart(); CacheLocator.getIndiciesCache().clearCache(); - assertNotNull(indiciesAPI.loadIndicies().getSiteSearch()); - assertTrue(indiciesAPI.loadIndicies().getSiteSearch().equals(indexName)); + assertNotNull(indicesAPI.loadLegacyIndices().getSiteSearch()); + assertTrue(indicesAPI.loadLegacyIndices().getSiteSearch().equals(indexName)); assertEquals(aliasName, indexAPI.getIndexAlias(indexName)); } finally { contentletIndexAPI.stopFullReindexation(); @@ -146,8 +143,8 @@ public void testReindexAbortKeepsDefaultSiteSearchIndex() indexTimestamp = contentletIndexAPI.fullReindexStart(); contentletIndexAPI.fullReindexAbort(); CacheLocator.getIndiciesCache().clearCache(); - assertNotNull(indiciesAPI.loadIndicies().getSiteSearch()); - assertTrue(indiciesAPI.loadIndicies().getSiteSearch().equals(indexName)); + assertNotNull(indicesAPI.loadLegacyIndices().getSiteSearch()); + assertTrue(indicesAPI.loadLegacyIndices().getSiteSearch().equals(indexName)); assertEquals(aliasName, indexAPI.getIndexAlias(indexName)); } finally { contentletIndexAPI.stopFullReindexation(); @@ -190,7 +187,7 @@ public void test_listIndices_defaultIndicesShouldFirst() throws IOException, Dot //get the list of indices final List indices =siteSearchAPI.listIndices(); //validate if the new default index is the first in list - assertTrue(indiciesAPI.loadIndicies().getSiteSearch().equals(defIndex)); + assertTrue(indicesAPI.loadLegacyIndices().getSiteSearch().equals(defIndex)); assertEquals(defIndex, indices.get(0)); } diff --git a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/IndiciesFactoryTest.java b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/IndicesFactoryTest.java similarity index 80% rename from dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/IndiciesFactoryTest.java rename to dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/IndicesFactoryTest.java index 3c2dbc3707dc..4600d24f2711 100644 --- a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/IndiciesFactoryTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/IndicesFactoryTest.java @@ -1,19 +1,19 @@ package com.dotcms.content.elasticsearch.business; -import static org.junit.Assert.*; - -import com.dotcms.content.elasticsearch.business.IndiciesInfo.Builder; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import com.dotcms.content.elasticsearch.business.LegacyIndicesInfo.Builder; import com.dotcms.util.IntegrationTestInitService; import com.dotmarketing.business.APILocator; import com.dotmarketing.business.CacheLocator; import com.dotmarketing.business.FactoryLocator; import com.dotmarketing.common.db.DotConnect; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; -public class IndiciesFactoryTest { +public class IndicesFactoryTest { private static final String INFO_REINDEX_WORKING = "info.reindex_working"; private static final String INFO_WORKING = "info.working"; @@ -21,26 +21,26 @@ public class IndiciesFactoryTest { private static final String INFO_REINDEX_LIVE = "info.reindex_live"; private static String ORIGINAL_WORKING_INDEX; private static String ORIGINAL_LIVE_INDEX; - static IndiciesFactory indiciesFactory; - static IndiciesAPI indiciesAPI; + static IndicesFactory indiciesFactory; + static IndicesAPI indicesAPI; @BeforeClass public static void prepare() throws Exception { //Setting web app environment IntegrationTestInitService.getInstance().init(); indiciesFactory = FactoryLocator.getIndiciesFactory(); - indiciesAPI = APILocator.getIndiciesAPI(); + indicesAPI = APILocator.getIndiciesAPI(); - IndiciesInfo indiciesInfo = indiciesAPI.loadIndicies(); - ORIGINAL_WORKING_INDEX = indiciesInfo.getWorking(); - ORIGINAL_LIVE_INDEX = indiciesInfo.getLive(); + IndicesInfo legacyIndicesInfo = indicesAPI.loadLegacyIndices(); + ORIGINAL_WORKING_INDEX = legacyIndicesInfo.getWorking(); + ORIGINAL_LIVE_INDEX = legacyIndicesInfo.getLive(); } @Test public void test_index_pointing_when_previously_null() throws Exception{ new DotConnect().setSQL("delete from indicies").loadResult(); CacheLocator.getIndiciesCache().clearCache(); - IndiciesInfo nullInfo = indiciesAPI.loadIndicies(); + IndicesInfo nullInfo = indicesAPI.loadLegacyIndices(); assert(nullInfo.getLive()==null); assert(nullInfo.getWorking()==null); assert(nullInfo.getReindexLive()==null); @@ -52,7 +52,7 @@ public void test_index_pointing_when_previously_null() throws Exception{ indiciesFactory.point(builder.build()); - final IndiciesInfo cachedInfo = indiciesAPI.loadIndicies(); + final IndicesInfo cachedInfo = indicesAPI.loadLegacyIndices(); assertEquals(cachedInfo.getLive(), INFO_LIVE); assertEquals(cachedInfo.getWorking(), INFO_WORKING); assertEquals(cachedInfo.getReindexLive(), INFO_REINDEX_LIVE); @@ -70,7 +70,7 @@ public void test_index_repointing_when_previously_set() throws Exception{ builder.setWorking(INFO_WORKING).setLive(INFO_LIVE); indiciesFactory.point(builder.build()); - IndiciesInfo cachedInfo = indiciesAPI.loadIndicies(); + IndicesInfo cachedInfo = indicesAPI.loadLegacyIndices(); assertEquals(cachedInfo.getLive(), INFO_LIVE); assertEquals(cachedInfo.getWorking(), INFO_WORKING); assertNull(cachedInfo.getReindexLive()); @@ -80,7 +80,7 @@ public void test_index_repointing_when_previously_set() throws Exception{ builder.setReindexLive(INFO_REINDEX_LIVE).setReindexWorking(INFO_REINDEX_WORKING); indiciesFactory.point(builder.build()); - cachedInfo = indiciesAPI.loadIndicies(); + cachedInfo = indicesAPI.loadLegacyIndices(); assertNull(cachedInfo.getLive()); assertNull(cachedInfo.getWorking()); assertEquals(cachedInfo.getReindexLive(), INFO_REINDEX_LIVE); diff --git a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/util/ESMappingUtilHelperTest.java b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/util/ESMappingUtilHelperTest.java index b4776dc3f940..4bbe33e23741 100644 --- a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/util/ESMappingUtilHelperTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/util/ESMappingUtilHelperTest.java @@ -329,7 +329,7 @@ public void testAddMappingForFields() Config.setProperty("CREATE_TEXT_INDEX_FIELD_FOR_NON_TEXT_FIELDS", true); //Create a working index - boolean result = contentletIndexAPI.createContentIndex(workingIndex); + boolean result = contentletIndexAPI.createContentIndexLegacy(workingIndex); //Validate assertTrue(result); @@ -422,7 +422,7 @@ public void testMappingForDateFields() workingIndex = IndexType.WORKING.getPrefix() + "_" + System.currentTimeMillis(); //Create a working index - boolean result = contentletIndexAPI.createContentIndex(workingIndex); + boolean result = contentletIndexAPI.createContentIndexLegacy(workingIndex); assertTrue(result); contentletIndexAPI.activateIndex(workingIndex); @@ -437,7 +437,7 @@ public void testMappingForDateFields() //verifies mapping type for common text fields Map mapping = (Map) esMappingAPI - .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadIndicies().getWorking(), + .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadLegacyIndices().getWorking(), contentType.variable().toLowerCase() + "." + dateField.variable() .toLowerCase()).entrySet() .iterator() @@ -447,7 +447,7 @@ public void testMappingForDateFields() assertEquals(formatExpected, mapping.get("format")); mapping = (Map) esMappingAPI - .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadIndicies().getWorking(), + .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadLegacyIndices().getWorking(), contentType.variable().toLowerCase() + "." + dateTimeField.variable() .toLowerCase()).entrySet() .iterator() @@ -457,7 +457,7 @@ public void testMappingForDateFields() assertEquals(formatExpected, mapping.get("format")); mapping = (Map) esMappingAPI - .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadIndicies().getWorking(), + .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadLegacyIndices().getWorking(), "moddate").entrySet().iterator().next().getValue(); assertTrue(UtilMethods.isSet(mapping.get("type"))); assertEquals("date", mapping.get("type")); @@ -546,7 +546,7 @@ public void testCreateContentIndexWithCustomMappings() throws Exception { workingIndex = new ESIndexAPI().getNameWithClusterIDPrefix(IndexType.WORKING.getPrefix() + "_" + timestamp); //Create a working index - boolean result = contentletIndexAPI.createContentIndex(workingIndex); + boolean result = contentletIndexAPI.createContentIndexLegacy(workingIndex); //Validate assertTrue(result); @@ -728,7 +728,7 @@ public void testValidateEventMapping(final String testCase, final String[] field //verifies analyzer for common text fields final Map mapping = (Map) esMappingAPI - .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadIndicies().getWorking(), + .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadLegacyIndices().getWorking(), "calendarevent.title").entrySet().iterator() .next().getValue(); assertTrue(UtilMethods.isSet(mapping.get("type"))); @@ -760,7 +760,7 @@ private void validateMappingForFields(final String testCase, final String conten Logger.info(this, "Validating mapping for case: " + testCase + ". Field Name: " + field); mapping = (Map) esMappingAPI - .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadIndicies().getWorking(), + .getFieldMappingAsMap(APILocator.getIndiciesAPI().loadLegacyIndices().getWorking(), expectedResult.equals("geo_point") ? contentTypeVarName + StringPool.PERIOD + field : field).entrySet().iterator() .next().getValue(); diff --git a/dotcms-integration/src/test/java/com/dotcms/content/opensearch/util/DotOpenSearchClientProviderIntegrationTest.java b/dotcms-integration/src/test/java/com/dotcms/content/opensearch/util/DotOpenSearchClientProviderIntegrationTest.java new file mode 100644 index 000000000000..0369274f016b --- /dev/null +++ b/dotcms-integration/src/test/java/com/dotcms/content/opensearch/util/DotOpenSearchClientProviderIntegrationTest.java @@ -0,0 +1,435 @@ +package com.dotcms.content.opensearch.util; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.dotcms.DataProviderWeldRunner; +import com.dotcms.IntegrationTestBase; +import com.dotcms.util.IntegrationTestInitService; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.util.Config; +import java.io.IOException; +import java.time.Duration; +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch.cluster.HealthRequest; +import org.opensearch.client.opensearch.cluster.HealthResponse; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.ExistsRequest; + +/** + * Integration test for ConfigurableOpenSearchProvider. + * Tests the provider's ability to create and configure OpenSearch clients. + * + * @author fabrizio + */ +@ApplicationScoped +@RunWith(DataProviderWeldRunner.class) +public class DotOpenSearchClientProviderIntegrationTest extends IntegrationTestBase { + + private static final String TEST_INDEX = "test-dotcms-opensearch-" + System.currentTimeMillis(); + + @Inject + OpenSearchDefaultClientProvider singleton; + + @BeforeClass + public static void prepare() throws Exception { + // Setting web app environment + IntegrationTestInitService.getInstance().init(); + + } + + /** + * Test direct provider with custom configuration + * This demonstrates how tests should use ConfigurableOpenSearchProvider directly for custom configs + */ + @Test + public void test_directProvider_withCustomConfiguration_shouldWork() { + ConfigurableOpenSearchProvider provider = null; + try { + // Arrange - Create test configuration + OpenSearchClientConfig testConfig = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9201") // Local OpenSearch port + .tlsEnabled(false) + .maxConnections(200) // Different configuration for testing + .maxConnectionsPerRoute(100) + .build(); + + // Act - Create provider with custom configuration + provider = new ConfigurableOpenSearchProvider(testConfig); + OpenSearchClient client = provider.getClient(); + + // Assert + assertNotNull("Provider should not be null", provider); + assertNotNull("Client should not be null", client); + + // Verify the singleton still works independently + OpenSearchClient singletonClient = singleton.getClient(); + assertNotNull("Singleton client should not be null", singletonClient); + + // They should be different instances (custom vs default config) + assertTrue("Should be different client instances", client != singletonClient); + + } finally { + closeProvider(provider); + } + } + + /** + * Test direct provider with convenience configurations + * Shows how to use the convenience configuration methods with ConfigurableOpenSearchProvider + */ + @Test + public void test_directProvider_withConvenienceConfigurations_shouldWork() { + ConfigurableOpenSearchProvider localProvider = null; + ConfigurableOpenSearchProvider prodProvider = null; + try { + // Test 1: Local test configuration + OpenSearchClientConfig localConfig = OpenSearchDefaultClientProvider.createLocalTestConfig(); + localProvider = new ConfigurableOpenSearchProvider(localConfig); + + assertEquals("Should use local port", "http://localhost:9201", localConfig.endpoints().get(0)); + assertFalse("Should have TLS disabled", localConfig.tlsEnabled()); + + // Test client creation + OpenSearchClient localClient = localProvider.getClient(); + assertNotNull("Local test client should not be null", localClient); + + // Test 2: Production-like test configuration + OpenSearchClientConfig prodConfig = OpenSearchDefaultClientProvider.createProductionTestConfig(); + prodProvider = new ConfigurableOpenSearchProvider(prodConfig); + + assertEquals("Should use production endpoint", "https://opensearch.prod.com:9200", prodConfig.endpoints().get(0)); + assertTrue("Should have TLS enabled", prodConfig.tlsEnabled()); + assertTrue("Should have username", prodConfig.username().isPresent()); + + // Test client creation + OpenSearchClient prodClient = prodProvider.getClient(); + assertNotNull("Production test client should not be null", prodClient); + + // Verify they are different instances + assertTrue("Should be different client instances", localClient != prodClient); + + } finally { + // Cleanup + closeProvider(localProvider); + closeProvider(prodProvider); + } + } + + /** + * Test configuration from properties + * This tests the properties loading mechanism using local OpenSearch + */ + @Test + public void test_loadConfiguration_fromProperties_shouldCreateValidConfig() { + // Arrange - Set some test properties for local OpenSearch + String originalEndpoints = Config.getStringProperty("OS_ENDPOINTS", null); + String originalTlsEnabled = Config.getStringProperty("OS_TLS_ENABLED", null); + + try { + // Set test properties for local OpenSearch (no auth, security disabled) + Config.setProperty("OS_ENDPOINTS", "http://localhost:9201"); + Config.setProperty("OS_TLS_ENABLED", false); + + // Act + ConfigurableOpenSearchProvider provider = new ConfigurableOpenSearchProvider(); + OpenSearchClient client = provider.getClient(); + + // Assert + assertNotNull("Provider should not be null", provider); + assertNotNull("Client should not be null", client); + + // Cleanup + closeProvider(provider); + + } finally { + // Reset properties + resetProperty("OS_ENDPOINTS", originalEndpoints); + resetProperty("OS_TLS_ENABLED", originalTlsEnabled); + } + } + + /** + * Test client functionality with cluster health check + * This test connects to the local OpenSearch instance on port 9201 + */ + @Test + public void test_clientFunctionality_clusterHealth_shouldReturnValidResponse() { + // This test connects to local OpenSearch (opensearch-3x container on port 9201) + ConfigurableOpenSearchProvider provider = null; + try { + // Arrange - Configure for local OpenSearch container + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9201") // Local OpenSearch port + .tlsEnabled(false) // Security disabled + .trustSelfSigned(true) + .connectionTimeout(Duration.ofSeconds(10)) // Increased timeout + .socketTimeout(Duration.ofSeconds(10)) + .build(); + + provider = new ConfigurableOpenSearchProvider(config); + OpenSearchClient client = provider.getClient(); + + // Act - Try to get cluster health + + HealthResponse healthResponse = client.cluster().health(); + + // Assert + assertNotNull("Health response should not be null", healthResponse); + assertNotNull("Cluster name should not be null", healthResponse.clusterName()); + + System.out.println("✅ OpenSearch cluster health: " + healthResponse.status()); + System.out.println("✅ Cluster name: " + healthResponse.clusterName()); + System.out.println("✅ Number of nodes: " + healthResponse.numberOfNodes()); + System.out.println("✅ Active shards: " + healthResponse.activeShards()); + + // Additional assertions for the specific container configuration + assertTrue("Should have at least one node", healthResponse.numberOfNodes() >= 1); + assertEquals("Should be single-node cluster", "opensearch-3x-cluster", healthResponse.clusterName()); + + } catch (OpenSearchException | IOException e) { + // OpenSearch is not available - provide helpful message + System.out.println("⚠️ Local OpenSearch not available for integration test: " + e.getMessage()); + System.out.println("⚠️ Make sure OpenSearch container is running with:"); + System.out.println("⚠️ docker-compose up opensearch-3x"); + System.out.println("⚠️ Expected container on http://localhost:9201"); + System.out.println("⚠️ Skipping cluster health test"); + } catch (Exception e) { + System.out.println("❌ Unexpected error during cluster health test: " + e.getMessage()); + e.printStackTrace(); + // Don't fail the test for unexpected errors in case OpenSearch is not available + } finally { + closeProvider(provider); + } + } + + /** + * Test index operations using local OpenSearch + */ + @Test + public void test_clientFunctionality_indexOperations_shouldWorkCorrectly() { + ConfigurableOpenSearchProvider provider = null; + try { + // Arrange - Configure for local OpenSearch container + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9201") // Local OpenSearch port + .tlsEnabled(false) // Security disabled + .trustSelfSigned(true) + .connectionTimeout(Duration.ofSeconds(10)) + .socketTimeout(Duration.ofSeconds(10)) + .build(); + + provider = new ConfigurableOpenSearchProvider(config); + OpenSearchClient client = provider.getClient(); + + // Act & Assert - Try basic index operations + + // 1. Check if test index exists (should not) + ExistsRequest existsRequest = ExistsRequest.of(e -> e.index(TEST_INDEX)); + boolean existsBefore = client.indices().exists(existsRequest).value(); + assertFalse("Test index should not exist initially", existsBefore); + System.out.println("✅ Verified test index does not exist initially: " + TEST_INDEX); + + // 2. Create test index + CreateIndexRequest createRequest = CreateIndexRequest.of(c -> c.index(TEST_INDEX)); + client.indices().create(createRequest); + System.out.println("✅ Created test index: " + TEST_INDEX); + + // 3. Check if test index exists now (should exist) + boolean existsAfter = client.indices().exists(existsRequest).value(); + assertTrue("Test index should exist after creation", existsAfter); + System.out.println("✅ Verified test index exists after creation"); + + // 4. Delete test index + DeleteIndexRequest deleteRequest = DeleteIndexRequest.of(d -> d.index(TEST_INDEX)); + client.indices().delete(deleteRequest); + System.out.println("✅ Deleted test index: " + TEST_INDEX); + + // 5. Verify index is deleted + boolean existsAfterDelete = client.indices().exists(existsRequest).value(); + assertFalse("Test index should not exist after deletion", existsAfterDelete); + System.out.println("✅ Verified test index does not exist after deletion"); + + System.out.println("✅ All OpenSearch index operations completed successfully!"); + + } catch (OpenSearchException | IOException e) { + // OpenSearch is not available - provide helpful message + System.out.println("⚠️ Local OpenSearch not available for index operations test: " + e.getMessage()); + System.out.println("⚠️ Make sure OpenSearch container is running with:"); + System.out.println("⚠️ docker-compose up opensearch-3x"); + System.out.println("⚠️ Expected container on http://localhost:9201"); + System.out.println("⚠️ Skipping index operations test"); + } catch (Exception e) { + System.out.println("❌ Unexpected error during index operations test: " + e.getMessage()); + e.printStackTrace(); + // Don't fail the test for unexpected errors in case OpenSearch is not available + } finally { + closeProvider(provider); + } + } + + /** + * Comprehensive connectivity test for local OpenSearch container + * This test provides detailed information about the OpenSearch cluster + */ + @Test + public void test_localOpenSearchConnectivity_shouldProvideDetailedInfo() { + ConfigurableOpenSearchProvider provider = null; + try { + // Arrange - Configure specifically for the opensearch-3x container + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("http://localhost:9201") // Container mapped port + .tlsEnabled(false) // DISABLE_SECURITY_PLUGIN=true + .connectionTimeout(Duration.ofSeconds(15)) + .socketTimeout(Duration.ofSeconds(15)) + .maxConnections(10) + .maxConnectionsPerRoute(5) + .build(); + + System.out.println("🔍 Testing connectivity to local OpenSearch container..."); + System.out.println("🔍 Endpoint: http://localhost:9201"); + System.out.println("🔍 Expected cluster: opensearch-3x-cluster"); + + provider = new ConfigurableOpenSearchProvider(config); + OpenSearchClient client = provider.getClient(); + + // Test 1: Basic cluster health + System.out.println("\n📊 Getting cluster health..."); + HealthRequest healthRequest = HealthRequest.of(h -> + h.timeout(new Time.Builder().time("15s").build()) + ); + HealthResponse healthResponse = client.cluster().health(healthRequest); + + // Detailed assertions and logging + assertNotNull("Health response should not be null", healthResponse); + System.out.println("✅ Health Status: " + healthResponse.status()); + System.out.println("✅ Cluster Name: " + healthResponse.clusterName()); + System.out.println("✅ Number of Nodes: " + healthResponse.numberOfNodes()); + System.out.println("✅ Active Shards: " + healthResponse.activeShards()); + System.out.println("✅ Initializing Shards: " + healthResponse.initializingShards()); + System.out.println("✅ Relocating Shards: " + healthResponse.relocatingShards()); + System.out.println("✅ Unassigned Shards: " + healthResponse.unassignedShards()); + + // Verify expected container configuration + assertEquals("Should match container cluster name", + "opensearch-3x-cluster", healthResponse.clusterName()); + assertEquals("Should have exactly one node (single-node setup)", + 1, healthResponse.numberOfNodes()); + + // Test 2: Get cluster info + System.out.println("\n🏗️ Getting cluster information..."); + try { + var infoResponse = client.info(); + System.out.println("✅ OpenSearch Version: " + infoResponse.version().number()); + System.out.println("✅ Lucene Version: " + infoResponse.version().luceneVersion()); + System.out.println("✅ Build Date: " + infoResponse.version().buildDate()); + System.out.println("✅ Build Hash: " + infoResponse.version().buildHash().substring(0, 8) + "..."); + + assertNotNull("Info response should not be null", infoResponse); + assertTrue("Should be OpenSearch 3.x", + infoResponse.version().number().startsWith("3.")); + } catch (Exception e) { + System.out.println("⚠️ Could not get cluster info: " + e.getMessage()); + } + + System.out.println("\n✅ Local OpenSearch connectivity test PASSED!"); + System.out.println("✅ Container opensearch-3x is running and accessible"); + + } catch (OpenSearchException e) { + System.out.println("\n❌ OpenSearch API Error:"); + System.out.println("❌ Error: " + e.getMessage()); + System.out.println("❌ Status: " + e.status()); + System.out.println("\n🔧 Troubleshooting steps:"); + System.out.println("🔧 1. Check if container is running: docker ps | grep opensearch"); + System.out.println("🔧 2. Check container logs: docker logs opensearch-3x"); + System.out.println("🔧 3. Verify port mapping: curl http://localhost:9201"); + fail("OpenSearch API error: " + e.getMessage()); + + } catch (IOException e) { + System.out.println("\n❌ Connection Error:"); + System.out.println("❌ " + e.getMessage()); + System.out.println("\n🔧 Troubleshooting steps:"); + System.out.println("🔧 1. Start OpenSearch container: docker-compose up -d opensearch-3x"); + System.out.println("🔧 2. Wait for startup: docker logs -f opensearch-3x"); + System.out.println("🔧 3. Test direct access: curl http://localhost:9201"); + System.out.println("🔧 4. Check network: docker network ls"); + + // Don't fail the test - just report the issue + System.out.println("\n⚠️ Skipping connectivity test - OpenSearch not available"); + + } catch (Exception e) { + System.out.println("\n❌ Unexpected error: " + e.getClass().getSimpleName()); + System.out.println("❌ Message: " + e.getMessage()); + e.printStackTrace(); + fail("Unexpected error during connectivity test: " + e.getMessage()); + + } finally { + closeProvider(provider); + } + } + + /** + * Test provider close functionality + */ + @Test + public void test_closeProvider_shouldNotThrowException() { + // Arrange + ConfigurableOpenSearchProvider provider = new ConfigurableOpenSearchProvider(); + + // Act & Assert - Should not throw exception + try { + provider.close(); + } catch (Exception e) { + fail("Close should not throw exception: " + e.getMessage()); + } + } + + /** + * Test invalid configuration scenarios + */ + @Test(expected = DotRuntimeException.class) + public void test_createProvider_withInvalidEndpoint_shouldThrowException() { + // Arrange + OpenSearchClientConfig config = OpenSearchClientConfig.builder() + .addEndpoints("invalid-url-format") + .build(); + + // Act & Assert - Should throw DotRuntimeException due to invalid URL + new ConfigurableOpenSearchProvider(config); + } + + /** + * Helper method to close provider safely + */ + private void closeProvider(ConfigurableOpenSearchProvider provider) { + if (provider != null) { + try { + provider.close(); + } catch (IOException e) { + System.err.println("Warning: Error closing provider: " + e.getMessage()); + } + } + } + + /** + * Helper method to reset properties + */ + private void resetProperty(String key, String originalValue) { + if (originalValue != null) { + Config.setProperty(key, originalValue); + } else { + Config.setProperty(key, ""); + } + } +} \ No newline at end of file diff --git a/dotcms-integration/src/test/java/com/dotcms/contenttype/business/FieldAPITest.java b/dotcms-integration/src/test/java/com/dotcms/contenttype/business/FieldAPITest.java index 162fcb4780cc..670c84efbfe7 100644 --- a/dotcms-integration/src/test/java/com/dotcms/contenttype/business/FieldAPITest.java +++ b/dotcms-integration/src/test/java/com/dotcms/contenttype/business/FieldAPITest.java @@ -1,8 +1,33 @@ package com.dotcms.contenttype.business; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.BASE_TYPE; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.CONTENT_TYPE; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.IDENTIFIER; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.INODE; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.LIVE; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.MOD_DATE; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.TITLE; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.URL_MAP; +import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.WORKING; +import static com.dotcms.contenttype.business.ContentTypeAPIImpl.TYPES_AND_FIELDS_VALID_VARIABLE_REGEX; +import static com.dotmarketing.portlets.contentlet.model.Contentlet.ARCHIVED_KEY; +import static com.dotmarketing.portlets.contentlet.model.Contentlet.FOLDER_KEY; +import static com.dotmarketing.portlets.contentlet.model.Contentlet.HOST_KEY; +import static com.dotmarketing.portlets.contentlet.model.Contentlet.LOCKED_KEY; +import static com.dotmarketing.portlets.contentlet.model.Contentlet.MOD_USER_KEY; +import static com.dotmarketing.portlets.contentlet.model.Contentlet.OWNER_KEY; +import static com.dotmarketing.portlets.contentlet.model.Contentlet.TITLE_IMAGE_KEY; +import static com.dotmarketing.util.WebKeys.Relationship.RELATIONSHIP_CARDINALITY.MANY_TO_ONE; +import static com.dotmarketing.util.WebKeys.Relationship.RELATIONSHIP_CARDINALITY.ONE_TO_MANY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import com.dotcms.IntegrationTestBase; import com.dotcms.content.elasticsearch.business.ESMappingAPIImpl; -import com.dotcms.content.elasticsearch.business.IndiciesInfo; +import com.dotcms.content.elasticsearch.business.IndicesInfo; import com.dotcms.contenttype.business.FieldAPITest.UniqueConstraintTestCase.DuplicateType; import com.dotcms.contenttype.model.field.BinaryField; import com.dotcms.contenttype.model.field.DateField; @@ -37,7 +62,6 @@ import com.dotmarketing.exception.DotDataException; import com.dotmarketing.exception.DotDataValidationException; import com.dotmarketing.exception.DotSecurityException; -import com.dotmarketing.portlets.contentlet.business.HostAPI; import com.dotmarketing.portlets.folders.business.FolderAPI; import com.dotmarketing.portlets.structure.model.Relationship; import com.dotmarketing.util.Config; @@ -51,42 +75,16 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import io.vavr.Tuple2; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; - -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.BASE_TYPE; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.CONTENT_TYPE; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.IDENTIFIER; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.INODE; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.LIVE; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.MOD_DATE; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.TITLE; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.URL_MAP; -import static com.dotcms.content.elasticsearch.constants.ESMappingConstants.WORKING; -import static com.dotcms.contenttype.business.ContentTypeAPIImpl.TYPES_AND_FIELDS_VALID_VARIABLE_REGEX; -import static com.dotmarketing.portlets.contentlet.model.Contentlet.ARCHIVED_KEY; -import static com.dotmarketing.portlets.contentlet.model.Contentlet.FOLDER_KEY; -import static com.dotmarketing.portlets.contentlet.model.Contentlet.HOST_KEY; -import static com.dotmarketing.portlets.contentlet.model.Contentlet.LOCKED_KEY; -import static com.dotmarketing.portlets.contentlet.model.Contentlet.MOD_USER_KEY; -import static com.dotmarketing.portlets.contentlet.model.Contentlet.OWNER_KEY; -import static com.dotmarketing.portlets.contentlet.model.Contentlet.TITLE_IMAGE_KEY; -import static com.dotmarketing.util.WebKeys.Relationship.RELATIONSHIP_CARDINALITY.MANY_TO_ONE; -import static com.dotmarketing.util.WebKeys.Relationship.RELATIONSHIP_CARDINALITY.ONE_TO_MANY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; @RunWith(DataProviderRunner.class) public class FieldAPITest extends IntegrationTestBase { @@ -1433,18 +1431,18 @@ public void test_SaveNewNoIndexedField_ShouldNotAddESMapping() .build(); field = fieldAPI.save(field, user); - final IndiciesInfo indiciesInfo = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo legacyIndicesInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); final ESMappingAPIImpl mappingAPI = new ESMappingAPIImpl(); //verify mapping on working index Map mapping = mappingAPI - .getFieldMappingAsMap(indiciesInfo.getWorking(), + .getFieldMappingAsMap(legacyIndicesInfo.getWorking(), (type.variable() + StringPool.PERIOD + field.variable())); assertFalse(UtilMethods.isSet(mapping)); //verify mapping on live index mapping = mappingAPI - .getFieldMappingAsMap(indiciesInfo.getLive(), + .getFieldMappingAsMap(legacyIndicesInfo.getLive(), (type.variable() + StringPool.PERIOD + field.variable())); assertFalse(UtilMethods.isSet(mapping)); }finally{ @@ -1473,12 +1471,12 @@ public void test_SaveNewIndexedField_ShouldAddESMapping() .build(); field = fieldAPI.save(field, user); - final IndiciesInfo indiciesInfo = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo legacyIndicesInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); final ESMappingAPIImpl mappingAPI = new ESMappingAPIImpl(); //verify mapping on working index Map mapping = (Map) mappingAPI - .getFieldMappingAsMap(indiciesInfo.getWorking(), + .getFieldMappingAsMap(legacyIndicesInfo.getWorking(), (type.variable() + StringPool.PERIOD + field.variable()) .toLowerCase()).get(field.variable()); assertTrue(UtilMethods.isSet(mapping.get("type"))); @@ -1486,7 +1484,7 @@ public void test_SaveNewIndexedField_ShouldAddESMapping() //verify mapping on live index mapping = (Map) mappingAPI - .getFieldMappingAsMap(indiciesInfo.getLive(), + .getFieldMappingAsMap(legacyIndicesInfo.getLive(), (type.variable() + StringPool.PERIOD + field.variable()) .toLowerCase()).get(field.variable()); assertTrue(UtilMethods.isSet(mapping.get("type"))); @@ -1494,7 +1492,7 @@ public void test_SaveNewIndexedField_ShouldAddESMapping() //validate _dotraw fields mapping = (Map) mappingAPI - .getFieldMappingAsMap(indiciesInfo.getLive(), + .getFieldMappingAsMap(legacyIndicesInfo.getLive(), (type.variable() + StringPool.PERIOD + field.variable() + "_dotraw") .toLowerCase()).get(field.variable() + "_dotraw"); assertTrue(UtilMethods.isSet(mapping.get("type"))); @@ -1521,12 +1519,12 @@ public void test_SaveNewRelationshipField_ShouldAddESMapping() final Field field = createAndSaveRelationshipField("newRel", type.id(), type.variable(), CARDINALITY); - final IndiciesInfo indiciesInfo = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo legacyIndicesInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); final ESMappingAPIImpl mappingAPI = new ESMappingAPIImpl(); //verify mapping on working index Map mapping = (Map) mappingAPI - .getFieldMappingAsMap(indiciesInfo.getWorking(), + .getFieldMappingAsMap(legacyIndicesInfo.getWorking(), (type.variable() + StringPool.PERIOD + field.variable()) .toLowerCase()).get(field.variable().toLowerCase()); assertTrue(UtilMethods.isSet(mapping.get("type"))); @@ -1534,7 +1532,7 @@ public void test_SaveNewRelationshipField_ShouldAddESMapping() //verify mapping on live index mapping = (Map) mappingAPI - .getFieldMappingAsMap(indiciesInfo.getLive(), + .getFieldMappingAsMap(legacyIndicesInfo.getLive(), (type.variable() + StringPool.PERIOD + field.variable()) .toLowerCase()).get(field.variable().toLowerCase()); assertTrue(UtilMethods.isSet(mapping.get("type"))); diff --git a/dotcms-integration/src/test/java/com/dotcms/security/apps/AppsAPIImplTest.java b/dotcms-integration/src/test/java/com/dotcms/security/apps/AppsAPIImplTest.java index d95a91cab111..ac37ac4cb569 100644 --- a/dotcms-integration/src/test/java/com/dotcms/security/apps/AppsAPIImplTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/security/apps/AppsAPIImplTest.java @@ -9,8 +9,8 @@ import static org.mockito.Mockito.when; import com.dotcms.content.elasticsearch.business.ContentletIndexAPI; -import com.dotcms.content.elasticsearch.business.IndiciesAPI; -import com.dotcms.content.elasticsearch.business.IndiciesInfo; +import com.dotcms.content.elasticsearch.business.IndicesAPI; +import com.dotcms.content.elasticsearch.business.IndicesInfo; import com.dotcms.datagen.AppDescriptorDataGen; import com.dotcms.datagen.LayoutDataGen; import com.dotcms.datagen.PortletDataGen; @@ -1525,8 +1525,8 @@ public void Test_NPE_Free_ToString(){ public void Test_AppKeyByHost_On_Index_Deactivation() throws DotDataException, IOException, DotSecurityException { final ContentletIndexAPI contentletIndexAPI = APILocator.getContentletIndexAPI(); - final IndiciesAPI indiciesAPI = APILocator.getIndiciesAPI(); - final IndiciesInfo indiciesInfo = indiciesAPI.loadIndicies(); + final IndicesAPI indicesAPI = APILocator.getIndiciesAPI(); + final IndicesInfo legacyIndicesInfo = indicesAPI.loadLegacyIndices(); try { AppSecrets.Builder builder = new AppSecrets.Builder(); @@ -1540,8 +1540,8 @@ public void Test_AppKeyByHost_On_Index_Deactivation() final Host host = new SiteDataGen().nextPersisted(); - contentletIndexAPI.deactivateIndex(indiciesInfo.getWorking()); - contentletIndexAPI.deactivateIndex(indiciesInfo.getLive()); + contentletIndexAPI.deactivateIndex(legacyIndicesInfo.getWorking()); + contentletIndexAPI.deactivateIndex(legacyIndicesInfo.getLive()); final AppsAPI api = APILocator.getAppsAPI(); api.saveSecrets(bean,host,APILocator.systemUser()); @@ -1550,8 +1550,8 @@ public void Test_AppKeyByHost_On_Index_Deactivation() assertTrue(keysByHost.get(host.getIdentifier()).contains(appKey.toLowerCase())); } finally { - contentletIndexAPI.activateIndex(indiciesInfo.getWorking()); - contentletIndexAPI.activateIndex(indiciesInfo.getLive()); + contentletIndexAPI.activateIndex(legacyIndicesInfo.getWorking()); + contentletIndexAPI.activateIndex(legacyIndicesInfo.getLive()); } } diff --git a/dotcms-integration/src/test/java/com/dotcms/util/pagination/ContentTypesPaginatorTest.java b/dotcms-integration/src/test/java/com/dotcms/util/pagination/ContentTypesPaginatorTest.java index 2cb14139e4a6..d3b4932fd96e 100644 --- a/dotcms-integration/src/test/java/com/dotcms/util/pagination/ContentTypesPaginatorTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/util/pagination/ContentTypesPaginatorTest.java @@ -1,18 +1,18 @@ package com.dotcms.util.pagination; -import com.dotcms.content.elasticsearch.business.IndiciesInfo; +import static com.liferay.util.StringPool.COMMA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.dotcms.content.elasticsearch.business.IndicesInfo; import com.dotcms.contenttype.business.ContentTypeAPI; import com.dotcms.contenttype.model.type.BaseContentType; import com.dotcms.contenttype.model.type.ContentType; import com.dotcms.contenttype.model.type.ContentTypeBuilder; import com.dotcms.languagevariable.business.LanguageVariableAPI; -import com.dotcms.util.CollectionsUtils; import com.dotcms.util.IntegrationTestInitService; import com.dotmarketing.beans.Host; -import com.dotmarketing.beans.Permission; import com.dotmarketing.business.APILocator; -import com.dotmarketing.business.PermissionAPI; -import com.dotmarketing.business.Role; import com.dotmarketing.exception.DotDataException; import com.dotmarketing.exception.DotSecurityException; import com.dotmarketing.portlets.folders.business.FolderAPI; @@ -24,23 +24,17 @@ import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; -import java.util.Objects; -import java.util.Set; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; - -import static com.liferay.util.StringPool.COMMA; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; /** * This Integration Test verifies that the {@link ContentTypesPaginator} class performs as expected. @@ -207,10 +201,10 @@ public void test_getItems_WhenFilterContainsContentTypeName_ReturnsTheContentTyp */ @Test public void whenTheIndicesAreDeactivateShouldReturnNAInEntries() throws DotDataException, IOException { - final IndiciesInfo indiciesInfo = APILocator.getIndiciesAPI().loadIndicies(); + final IndicesInfo legacyIndicesInfo = APILocator.getIndiciesAPI().loadLegacyIndices(); - final String live = APILocator.getESIndexAPI().removeClusterIdFromName(indiciesInfo.getLive()); - final String working = APILocator.getESIndexAPI().removeClusterIdFromName(indiciesInfo.getWorking()); + final String live = APILocator.getESIndexAPI().removeClusterIdFromName(legacyIndicesInfo.getLive()); + final String working = APILocator.getESIndexAPI().removeClusterIdFromName(legacyIndicesInfo.getWorking()); try { if (live != null) { APILocator.getContentletIndexAPI().deactivateIndex(live);