diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/BaseSegmentPushJobRunner.java b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/BaseSegmentPushJobRunner.java index 821e42d2568d..d45032a9ccd9 100644 --- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/BaseSegmentPushJobRunner.java +++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/BaseSegmentPushJobRunner.java @@ -18,7 +18,6 @@ */ package org.apache.pinot.plugin.ingestion.batch.common; -import com.google.common.base.Preconditions; import java.io.File; import java.io.IOException; import java.net.URI; @@ -113,8 +112,8 @@ public List getSegmentsToReplace(Map segmentsUriToTarPat for (String tarFilePath : tarFilePaths) { File tarFile = new File(tarFilePath); String fileName = tarFile.getName(); - Preconditions.checkArgument(fileName.endsWith(Constants.TAR_GZ_FILE_EXT)); - String segmentName = fileName.substring(0, fileName.length() - Constants.TAR_GZ_FILE_EXT.length()); + String segmentName = fileName.endsWith(Constants.TAR_GZ_FILE_EXT) + ? fileName.substring(0, fileName.length() - Constants.TAR_GZ_FILE_EXT.length()) : fileName; segmentNames.add(segmentName); } return segmentNames; diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java index 6ca93f24918e..eadd908b8717 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java @@ -18,7 +18,6 @@ */ package org.apache.pinot.segment.local.utils; -import com.google.common.base.Preconditions; import java.io.File; import java.io.InputStream; import java.io.Serializable; @@ -152,8 +151,8 @@ public static void pushSegments(SegmentGenerationJobSpec spec, PinotFS fileSyste URI tarFileURI = URI.create(tarFilePath); File tarFile = new File(tarFilePath); String fileName = tarFile.getName(); - Preconditions.checkArgument(fileName.endsWith(Constants.TAR_GZ_FILE_EXT)); - String segmentName = fileName.substring(0, fileName.length() - Constants.TAR_GZ_FILE_EXT.length()); + String segmentName = fileName.endsWith(Constants.TAR_GZ_FILE_EXT) + ? fileName.substring(0, fileName.length() - Constants.TAR_GZ_FILE_EXT.length()) : fileName; for (PinotClusterSpec pinotClusterSpec : spec.getPinotClusterSpecs()) { URI controllerURI; try { @@ -370,6 +369,7 @@ public static Map getSegmentUriToTarPathMap(URI outputDirURI, Pu for (String file : files) { if (pushFilePathMatcher != null) { if (!pushFilePathMatcher.matches(Paths.get(file))) { + LOGGER.info("Ignoring file {}", file); continue; } } @@ -379,11 +379,11 @@ public static Map getSegmentUriToTarPathMap(URI outputDirURI, Pu // Skip segment metadata tar gz files continue; } - if (uri.getPath().endsWith(Constants.TAR_GZ_FILE_EXT)) { - URI updatedURI = SegmentPushUtils.generateSegmentTarURI(outputDirURI, uri, pushSpec.getSegmentUriPrefix(), - pushSpec.getSegmentUriSuffix()); - segmentUriToTarPathMap.put(updatedURI.toString(), file); - } + //if (uri.getPath().endsWith(Constants.TAR_GZ_FILE_EXT)) { + URI updatedURI = SegmentPushUtils.generateSegmentTarURI(outputDirURI, uri, pushSpec.getSegmentUriPrefix(), + pushSpec.getSegmentUriSuffix()); + segmentUriToTarPathMap.put(updatedURI.toString(), file); + //} } return segmentUriToTarPathMap; }