Skip to content

Commit 222bf7d

Browse files
jsonwanjsonwan
authored
Merge pull request #4112 from TencentBlueKing/3.12.x
Merge branch 3.12.x into master
2 parents b43ba8e + 50bd606 commit 222bf7d

File tree

10 files changed

+60
-27
lines changed

10 files changed

+60
-27
lines changed

‎src/backend/build.gradle‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,8 @@ ext {
144144
set('commonsValidatorVersion', "1.6")
145145
set('okHttpVersion', "4.12.0")
146146
set('jcommanderVersion', "1.72")
147-
set('kubernetesJavaClientVersion', "11.0.4")
148-
set('springCloudKubernetesVersion', "2.0.6")
147+
set('kubernetesJavaClientVersion', "13.0.2")
148+
set('springCloudKubernetesVersion', "2.1.9")
149149
set('cryptoJavaSDKVersion', "1.1.3")
150150
// Fix CVE-2019-10086,CVE-2014-0114
151151
set('commonsBeanutilsVersion', "1.9.4")
@@ -294,7 +294,6 @@ subprojects {
294294
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
295295
mavenBom "org.springframework.cloud:spring-cloud-sleuth-otel-dependencies:${springCloudOtelVersion}"
296296
}
297-
// 由于spring cloud (2021.0.5)的 kubernetes 组件在特性版本jdk(8u252）下会出现不兼容（http2协议)，并且由于历史原因jdk版本无法升级，所以把spring cloud kubernetes 降低版本
298297
dependency "org.springframework.cloud:spring-cloud-kubernetes-client-autoconfig:$springCloudKubernetesVersion"
299298
dependency "org.springframework.cloud:spring-cloud-kubernetes-client-config:$springCloudKubernetesVersion"
300299
dependency "org.springframework.cloud:spring-cloud-kubernetes-client-discovery:$springCloudKubernetesVersion"

‎src/backend/commons/common-log/src/main/resources/logback-default.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,11 @@ Default job logback configuration provided for import
276276
</root>
277277
</springProfile>
278278
<springProfile name="dev &amp; kubernetes">
279+
<logger name="io.kubernetes.client.informer.cache.ReflectorRunnable" level="DEBUG" additivity="false" >
280+
<appender-ref ref="CONSOLE"/>
281+
<appender-ref ref="async-app-appender"/>
282+
<appender-ref ref="error-appender"/>
283+
</logger>
279284
<logger name="com.tencent.bk.job" level="DEBUG" additivity="false" >
280285
<appender-ref ref="CONSOLE"/>
281286
<appender-ref ref="async-app-appender"/>
@@ -321,6 +326,11 @@ Default job logback configuration provided for import
321326
</root>
322327
</springProfile>
323328
<springProfile name="prod &amp; kubernetes">
329+
<logger name="io.kubernetes.client.informer.cache.ReflectorRunnable" level="DEBUG" additivity="false" >
330+
<appender-ref ref="CONSOLE"/>
331+
<appender-ref ref="async-app-appender"/>
332+
<appender-ref ref="error-appender"/>
333+
</logger>
324334
<logger name="com.tencent.bk.job" level="INFO" additivity="false" >
325335
<appender-ref ref="CONSOLE"/>
326336
<appender-ref ref="async-app-appender"/>

‎src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicHostDTO.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ public class BasicHostDTO {
4444
* 主机ID
4545
*/
4646
private Long hostId;
47+
/**
48+
* 业务ID
49+
*/
50+
private Long bizId;
4751
/**
4852
* CMDB数据的上次修改时间
4953
*/

‎src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/background/event/cmdb/handler/HostEventHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ private void handleCreateOrUpdateEvent(ResourceEvent<HostEventDetail> event, App
154154
// 尝试设置Agent状态
155155
Integer agentStatus = tryToUpdateAgentStatus(hostInfoDTO);
156156
// 更新DB与缓存中的主机数据
157-
Pair<Boolean, Integer> pair = noTenantHostService.createOrUpdateHostBeforeLastTime(hostInfoDTO);
157+
Pair<Boolean, Integer> pair = noTenantHostService.createOrUpdateHostBeforeOrEqualLastTime(hostInfoDTO);
158158
int affectedNum = pair.getRight();
159159
if (affectedNum > 0) {
160160
log.info("{} host affected, created:{}", affectedNum, pair.getLeft());

‎src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/background/sync/BizHostSyncService.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,16 @@
3030
import com.tencent.bk.job.common.cc.model.result.ModuleProp;
3131
import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient;
3232
import com.tencent.bk.job.common.constant.CcNodeTypeEnum;
33+
import com.tencent.bk.job.common.constant.JobConstants;
3334
import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum;
3435
import com.tencent.bk.job.common.model.dto.ApplicationDTO;
3536
import com.tencent.bk.job.common.model.dto.ApplicationHostDTO;
3637
import com.tencent.bk.job.common.model.dto.BasicHostDTO;
3738
import com.tencent.bk.job.common.model.dto.ResourceScope;
3839
import com.tencent.bk.job.common.util.StringUtil;
3940
import com.tencent.bk.job.common.util.TimeUtil;
40-
import com.tencent.bk.job.manage.dao.ApplicationDAO;
4141
import com.tencent.bk.job.common.watch.SafeWatch;
42+
import com.tencent.bk.job.manage.dao.ApplicationDAO;
4243
import com.tencent.bk.job.manage.dao.HostTopoDAO;
4344
import com.tencent.bk.job.manage.dao.NoTenantHostDAO;
4445
import com.tencent.bk.job.manage.model.dto.HostTopoDTO;
@@ -213,15 +214,29 @@ private boolean needToUpdate(HostProp hostProp,
213214
);
214215
}
215216
BasicHostDTO localHost = localHostsIdMap.get(hostProp.getHostId());
216-
if (localHost.getLastTime() >= lastTime) {
217+
if (localHost.getLastTime() > lastTime) {
218+
// 本地主机的时间戳更新，不需要更新
217219
log.debug(
218-
"local host(hostId={}, lastTime={}) is not older than target host last time({}), ignore update",
220+
"local host(hostId={}, lastTime={}) is newer than target host last time({}), ignore update",
219221
localHost.getHostId(),
220222
localHost.getLastTime(),
221223
cmdbHostsFetchTimeMills
222224
);
223225
return false;
226+
} else if (lastTime.equals(localHost.getLastTime())) {
227+
// 本地主机的时间戳与CMDB主机一致，但业务ID字段无有效值，需要更新
228+
if (localHost.getBizId() == JobConstants.PUBLIC_APP_ID) {
229+
log.info(
230+
"local host(hostId={}, lastTime={}) hasInvalidAppId, need to update",
231+
localHost.getHostId(),
232+
localHost.getLastTime()
233+
);
234+
return true;
235+
}
236+
// 本地主机的时间戳与CMDB主机一致，且业务ID字段有有效值，不需要更新
237+
return false;
224238
} else {
239+
// 本地主机的时间戳更旧，需要更新
225240
log.info(
226241
"local host(hostId={}, lastTime={}) is older than target host last time({}), need to update",
227242
localHost.getHostId(),
@@ -259,6 +274,7 @@ private Set<BasicHostDTO> refreshBizHostAndRelations(Long bizId,
259274
}
260275
BasicHostDTO cmdbBasicHost = new BasicHostDTO(
261276
hostProp.getHostId(),
277+
bizId,
262278
lastTimeMills
263279
);
264280
cmdbBasicHosts.add(cmdbBasicHost);
@@ -739,7 +755,7 @@ private int applyInsertHostList(List<ApplicationHostDTO> insertHostList,
739755

740756
private int tryToInsertOneHost(ApplicationHostDTO hostDTO) {
741757
try {
742-
Pair<Boolean, Integer> pair = noTenantHostService.createOrUpdateHostBeforeLastTime(hostDTO);
758+
Pair<Boolean, Integer> pair = noTenantHostService.createOrUpdateHostBeforeOrEqualLastTime(hostDTO);
743759
return pair.getRight();
744760
} catch (Exception e) {
745761
FormattingTuple msg = MessageFormatter.format("Fail to insert host={}", hostDTO);
@@ -754,7 +770,7 @@ private int applyUpdateHostList(List<ApplicationHostDTO> updateHostList,
754770
try {
755771
watch.start("batchUpdateHostsBeforeLastTime");
756772
// 更新主机
757-
updateNum = noTenantHostService.batchUpdateHostsBeforeLastTime(updateHostList);
773+
updateNum = noTenantHostService.batchUpdateHostsBeforeOrEqualLastTime(updateHostList);
758774
watch.stop();
759775
} catch (Exception e) {
760776
log.error("Fail to batchUpdateHostsBeforeLastTime, try to update one by one", e);
@@ -769,7 +785,7 @@ private int applyUpdateHostList(List<ApplicationHostDTO> updateHostList,
769785

770786
private int tryToUpdateOneHost(ApplicationHostDTO hostDTO) {
771787
try {
772-
return noTenantHostService.updateHostAttrsBeforeLastTime(hostDTO);
788+
return noTenantHostService.updateHostAttrsBeforeOrEqualLastTime(hostDTO);
773789
} catch (Exception e) {
774790
FormattingTuple msg = MessageFormatter.format(
775791
"Fail to updateHostAttrsBeforeLastTime, host={}",

‎src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/NoTenantHostDAO.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,9 +180,9 @@ Long countHostInfoByMultiKeys(Collection<Long> bizIds,
180180

181181
int updateHostAttrsByHostId(ApplicationHostDTO applicationHostDTO);
182182

183-
int updateHostAttrsBeforeLastTime(ApplicationHostDTO applicationHostDTO);
183+
int updateHostAttrsBeforeOrEqualLastTime(ApplicationHostDTO applicationHostDTO);
184184

185-
int batchUpdateHostsBeforeLastTime(List<ApplicationHostDTO> applicationHostDTOList);
185+
int batchUpdateHostsBeforeOrEqualLastTime(List<ApplicationHostDTO> applicationHostDTOList);
186186

187187
int syncHostTopo(Long hostId);
188188

‎src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/AbstractBaseHostDAO.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
import org.jooq.Field;
4646
import org.jooq.Record;
4747
import org.jooq.Record1;
48-
import org.jooq.Record2;
48+
import org.jooq.Record3;
4949
import org.jooq.Result;
5050
import org.jooq.SelectLimitStep;
5151
import org.jooq.SelectSeekStep1;
@@ -105,6 +105,7 @@ abstract public class AbstractBaseHostDAO {
105105

106106
protected static final TableField<?, ?>[] BASIC_FIELDS = {
107107
TABLE.HOST_ID,
108+
TABLE.APP_ID,
108109
TABLE.LAST_TIME
109110
};
110111

@@ -505,16 +506,18 @@ protected List<BasicHostDTO> listBasicHostInfoByConditions(Collection<Condition>
505506
}
506507
val query = context.select(
507508
TABLE.HOST_ID,
509+
TABLE.APP_ID,
508510
TABLE.LAST_TIME
509511
).from(TABLE)
510512
.where(conditions);
511-
Result<Record2<ULong, Long>> records = query.fetch();
513+
Result<Record3<ULong, ULong, Long>> records = query.fetch();
512514
List<BasicHostDTO> basicHostInfoList = new ArrayList<>();
513515

514516
if (CollectionUtils.isNotEmpty(records)) {
515517
records.forEach(record -> {
516518
BasicHostDTO basicHost = new BasicHostDTO(
517519
JooqDataTypeUtil.getLongFromULong(record.get(TABLE.HOST_ID)),
520+
JooqDataTypeUtil.getLongFromULong(record.get(TABLE.APP_ID)),
518521
record.get(TABLE.LAST_TIME)
519522
);
520523
basicHostInfoList.add(basicHost);
@@ -650,6 +653,7 @@ protected BasicHostDTO extractBasicHost(Record record) {
650653
}
651654
BasicHostDTO basicHost = new BasicHostDTO();
652655
basicHost.setHostId(record.get(TABLE.HOST_ID).longValue());
656+
basicHost.setBizId(record.get(TABLE.APP_ID).longValue());
653657
basicHost.setLastTime(record.get(TABLE.LAST_TIME));
654658
return basicHost;
655659
}

‎src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/NoTenantHostDAOImpl.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -483,11 +483,11 @@ public int updateHostAttrsByHostId(ApplicationHostDTO applicationHostDTO) {
483483
}
484484

485485
@Override
486-
public int updateHostAttrsBeforeLastTime(ApplicationHostDTO applicationHostDTO) {
486+
public int updateHostAttrsBeforeOrEqualLastTime(ApplicationHostDTO applicationHostDTO) {
487487
checkHostId(applicationHostDTO);
488488
List<Condition> conditions = new ArrayList<>();
489489
conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId())));
490-
conditions.add(TABLE.LAST_TIME.lessThan(applicationHostDTO.getLastTime()));
490+
conditions.add(TABLE.LAST_TIME.lessOrEqual(applicationHostDTO.getLastTime()));
491491
return updateHostAttrsByConditions(applicationHostDTO, conditions);
492492
}
493493

@@ -552,7 +552,7 @@ private Query buildQueryWithHostAndConditions(ApplicationHostDTO applicationHost
552552
}
553553

554554
@Override
555-
public int batchUpdateHostsBeforeLastTime(List<ApplicationHostDTO> hostList) {
555+
public int batchUpdateHostsBeforeOrEqualLastTime(List<ApplicationHostDTO> hostList) {
556556
int batchSize = 1000;
557557
int size = hostList.size();
558558
int start = 0;
@@ -566,7 +566,7 @@ public int batchUpdateHostsBeforeLastTime(List<ApplicationHostDTO> hostList) {
566566
for (ApplicationHostDTO host : subList) {
567567
List<Condition> conditions = new ArrayList<>();
568568
conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(host.getHostId())));
569-
conditions.add(TABLE.LAST_TIME.lessThan(host.getLastTime()));
569+
conditions.add(TABLE.LAST_TIME.lessOrEqual(host.getLastTime()));
570570
queryList.add(buildQueryWithHostAndConditions(host, conditions));
571571
}
572572
int[] results = context.batch(queryList).execute();

‎src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/NoTenantHostService.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,15 @@ public interface NoTenantHostService {
6161
* @param hostInfoList 主机信息
6262
* @return 成功更新的主机数量
6363
*/
64-
int batchUpdateHostsBeforeLastTime(List<ApplicationHostDTO> hostInfoList);
64+
int batchUpdateHostsBeforeOrEqualLastTime(List<ApplicationHostDTO> hostInfoList);
6565

6666
/**
6767
* 创建或更新主机（仅更新时间戳在当前数据之前的数据）
6868
*
6969
* @param hostInfoDTO 主机信息
7070
* @return Pair<是否创建 ， 受影响主机数量>
7171
*/
72-
Pair<Boolean, Integer> createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO);
72+
Pair<Boolean, Integer> createOrUpdateHostBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO);
7373

7474
int updateHostAttrsByHostId(ApplicationHostDTO hostInfoDTO);
7575

@@ -79,7 +79,7 @@ public interface NoTenantHostService {
7979
* @param hostInfoDTO 主机信息
8080
* @return 成功更新的主机数量
8181
*/
82-
int updateHostAttrsBeforeLastTime(ApplicationHostDTO hostInfoDTO);
82+
int updateHostAttrsBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO);
8383

8484
/**
8585
* 删除主机（仅删除时间戳等于或在当前数据之前的数据）

‎src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/NoTenantHostServiceImpl.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,14 @@ public int batchInsertHosts(List<ApplicationHostDTO> insertList) {
107107
}
108108

109109
@Override
110-
public int batchUpdateHostsBeforeLastTime(List<ApplicationHostDTO> hostInfoList) {
110+
public int batchUpdateHostsBeforeOrEqualLastTime(List<ApplicationHostDTO> hostInfoList) {
111111
if (CollectionUtils.isEmpty(hostInfoList)) {
112112
return 0;
113113
}
114114
StopWatch watch = new StopWatch();
115115
watch.start("batchUpdateHostsBeforeLastTime to DB");
116116
// 批量更新主机
117-
int affectedNum = noTenantHostDAO.batchUpdateHostsBeforeLastTime(hostInfoList);
117+
int affectedNum = noTenantHostDAO.batchUpdateHostsBeforeOrEqualLastTime(hostInfoList);
118118
log.info("try to update {} hosts, {} updated", hostInfoList.size(), affectedNum);
119119
watch.stop();
120120
watch.start("listHostInfoByHostIds");
@@ -227,12 +227,12 @@ private void loadDBHostToCache(List<HostSimpleDTO> simpleHostList) {
227227

228228
@JobTransactional(transactionManager = "jobManageTransactionManager")
229229
@Override
230-
public Pair<Boolean, Integer> createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO) {
230+
public Pair<Boolean, Integer> createOrUpdateHostBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO) {
231231
boolean needToCreate = false;
232232
try {
233233
if (noTenantHostDAO.existAppHostInfoByHostId(hostInfoDTO.getHostId())) {
234234
// 只更新事件中的主机属性与agent状态
235-
int affectedNum = noTenantHostDAO.updateHostAttrsBeforeLastTime(hostInfoDTO);
235+
int affectedNum = noTenantHostDAO.updateHostAttrsBeforeOrEqualLastTime(hostInfoDTO);
236236
if (affectedNum == 0) {
237237
ApplicationHostDTO hostInDB = noTenantHostDAO.getHostById(hostInfoDTO.getHostId());
238238
if (hostInDB != null) {
@@ -273,8 +273,8 @@ public int updateHostAttrsByHostId(ApplicationHostDTO hostInfoDTO) {
273273
return noTenantHostDAO.updateHostAttrsByHostId(hostInfoDTO);
274274
}
275275

276-
public int updateHostAttrsBeforeLastTime(ApplicationHostDTO hostInfoDTO) {
277-
return noTenantHostDAO.updateHostAttrsBeforeLastTime(hostInfoDTO);
276+
public int updateHostAttrsBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO) {
277+
return noTenantHostDAO.updateHostAttrsBeforeOrEqualLastTime(hostInfoDTO);
278278
}
279279

280280
@Override

0 commit comments

Comments
 (0)