Skip to content

Commit 9d6bacc

Browse files
yunyezhang-workyunyezhang-work
committed
RANGER-5406 Support export policies in a segmented manner
1 parent 0f980ba commit 9d6bacc

File tree

3 files changed

+84
-1
lines changed

3 files changed

+84
-1
lines changed

‎agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ public class SearchFilter {
5050
public static final String CREATE_TIME = "createTime"; // sort
5151
public static final String UPDATE_TIME = "updateTime"; // sort
5252
public static final String START_INDEX = "startIndex";
53+
public static final String BEGIN_INDEX = "beginIndex";
54+
public static final String OFFSET = "offset";
5355
public static final String PAGE_SIZE = "pageSize";
5456
public static final String SORT_BY = "sortBy";
5557
public static final String RESOURCE_SIGNATURE = "resourceSignature:"; // search
@@ -152,6 +154,8 @@ public class SearchFilter {
152154
private Map<String, Object[]> multiValueParams;
153155
private int startIndex;
154156
private int maxRows = Integer.MAX_VALUE;
157+
private int beginIndex = 0;
158+
private int offset = -1;
155159
private boolean getCount = true;
156160
private String sortBy;
157161
private String sortType;
@@ -167,6 +171,8 @@ public SearchFilter(SearchFilter other) {
167171
setMultiValueParams(other.multiValueParams != null ? new HashMap<>(other.multiValueParams) : null);
168172
setStartIndex(other.startIndex);
169173
setMaxRows(other.maxRows);
174+
setBeginIndex(other.beginIndex);
175+
setOffset(other.offset);
170176
setGetCount(other.getCount);
171177
setSortBy(other.sortBy);
172178
setSortType(other.sortType);
@@ -272,6 +278,23 @@ public void setStartIndex(int startIndex) {
272278
this.startIndex = startIndex;
273279
}
274280

281+
public int getBeginIndex() {
282+
return beginIndex;
283+
}
284+
285+
public void setBeginIndex(int beginIndex) {
286+
this.beginIndex = beginIndex;
287+
}
288+
289+
public int getOffset() {
290+
return offset;
291+
}
292+
293+
public void setOffset(int offset) {
294+
this.offset = offset;
295+
}
296+
297+
275298
public int getMaxRows() {
276299
return maxRows;
277300
}

‎security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,14 +239,30 @@ public SearchFilter extractCommonCriteriasForFilter(HttpServletRequest request,
239239
"Invalid value for parameter startIndex", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.START_INDEX);
240240

241241
startIndex = startIndex < 0 ? 0 : startIndex;
242+
logger.info("==> setStartIndex={}" , startIndex);
242243

243244
ret.setStartIndex(startIndex);
244245

245246
int pageSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.PAGE_SIZE), configUtil.getDefaultMaxRows(),
246247
"Invalid value for parameter pageSize", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.PAGE_SIZE);
247-
248+
logger.info("==> setMaxRows={}" , pageSize);
249+
logger.info("==> DefaultMaxRows={}" , configUtil.getDefaultMaxRows());
248250
ret.setMaxRows(validatePageSize(pageSize));
249251

252+
int beginIndex = restErrorUtil.parseInt(request.getParameter(SearchFilter.BEGIN_INDEX), 0,
253+
"Invalid value for parameter beginIndex", MessageEnums.INVALID_INPUT_DATA, null,
254+
SearchFilter.BEGIN_INDEX);
255+
beginIndex = beginIndex < 0 ? startIndex : beginIndex;
256+
logger.info("==> setBeginIndex={}" , beginIndex);
257+
ret.setBeginIndex(beginIndex);
258+
259+
int offsetSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.OFFSET), 0,
260+
"Invalid value for parameter offset", MessageEnums.INVALID_INPUT_DATA, null,
261+
SearchFilter.OFFSET);
262+
logger.info("==> setOffsetIndex={}" , offsetSize);
263+
offsetSize = offsetSize < 0 ? pageSize : offsetSize;
264+
ret.setOffset(offsetSize);
265+
250266
if (request.getParameter(SearchFilter.POLICY_TYPE) != null) {
251267
int policyType = restErrorUtil.parseInt(request.getParameter(SearchFilter.POLICY_TYPE), 0,
252268
"Invalid value for parameter policyType", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.POLICY_TYPE);

‎security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@
152152
import java.util.ArrayList;
153153
import java.util.Arrays;
154154
import java.util.Collection;
155+
import java.util.Collections;
155156
import java.util.Comparator;
156157
import java.util.HashMap;
157158
import java.util.HashSet;
@@ -3729,6 +3730,8 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
37293730
List<String> serviceTypeList = null;
37303731
List<String> serviceNameInServiceTypeList = new ArrayList<>();
37313732
boolean isServiceExists;
3733+
int begin = 0;
3734+
int offset = -1;
37323735

37333736
if (request.getParameter(PARAM_SERVICE_NAME) != null) {
37343737
serviceNames = request.getParameter(PARAM_SERVICE_NAME);
@@ -3750,6 +3753,9 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
37503753
List<RangerPolicy> policyListByServiceName = new ArrayList<>();
37513754

37523755
if (filter != null) {
3756+
begin = filter.getBeginIndex();
3757+
offset = filter.getOffset();
3758+
LOG.info("==> beginIndex: {}, offset: {}", begin, offset);
37533759
filter.setStartIndex(0);
37543760
filter.setMaxRows(Integer.MAX_VALUE);
37553761

@@ -3830,7 +3836,12 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
38303836
policyLists.addAll(orderedPolicies.values());
38313837
}
38323838
}
3839+
LOG.info("<==policyLists size:{}", policyLists.size());
38333840

3841+
if(begin>=0 && offset >0) {
3842+
policyLists = getRangerPoliciesInRange(policyLists, filter);
3843+
LOG.info("<==policyLists size after cut:{}" , policyLists.size());
3844+
}
38343845
return policyLists;
38353846
}
38363847

@@ -4298,6 +4309,39 @@ private RangerPolicyList toRangerPolicyList(List<RangerPolicy> policyList, Searc
42984309
return ret;
42994310
}
43004311

4312+
private List<RangerPolicy> getRangerPoliciesInRange(List<RangerPolicy> policyList, SearchFilter filter) {
4313+
if (CollectionUtils.isEmpty(policyList)) {
4314+
return Collections.emptyList();
4315+
}
4316+
4317+
int totalCount = policyList.size();
4318+
int startIndex = filter.getBeginIndex();
4319+
int offset = filter.getOffset();
4320+
int toIndex = Math.min(startIndex + offset, totalCount);
4321+
LOG.info("==>totalCount: {}, startIndex:{}, offsetSize: {}, toIndex: {}" , totalCount, startIndex, offset, toIndex);
4322+
String sortType = filter.getSortType();
4323+
String sortBy = filter.getSortBy();
4324+
4325+
if (StringUtils.isNotEmpty(sortBy) && StringUtils.isNotEmpty(sortType)) {
4326+
// By default policyList is sorted by policyId in asc order, So handling only desc case.
4327+
if (SearchFilter.POLICY_ID.equalsIgnoreCase(sortBy)) {
4328+
if (SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
4329+
policyList.sort(this.getPolicyComparator(sortBy, sortType));
4330+
}
4331+
} else if (SearchFilter.POLICY_NAME.equalsIgnoreCase(sortBy)) {
4332+
if (SORT_ORDER.ASC.name().equalsIgnoreCase(sortType) || SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
4333+
policyList.sort(this.getPolicyComparator(sortBy, sortType));
4334+
} else {
4335+
LOG.info("Invalid or Unsupported sortType : {}", sortType);
4336+
}
4337+
} else {
4338+
LOG.info("Invalid or Unsupported sortBy property : {}" , sortBy);
4339+
}
4340+
}
4341+
4342+
return new ArrayList<>(policyList.subList(startIndex, toIndex));
4343+
}
4344+
43014345
private Comparator<RangerPolicy> getPolicyComparator(String sortBy, String sortType) {
43024346
return (RangerPolicy me, RangerPolicy other) -> {
43034347
int ret = 0;

0 commit comments

Comments
 (0)