Skip to content

Commit d31f77c

Browse files
mikeaaaaaamikeaaaaaa
authored
Fit python nacos plugin (#281)
* [fit] add python nacos plugin. * [fit] Upgrade some plugin configurations to system configurations * [fit] 解决在同步函数中执行异步函数问题 * [fit] 修改代码bug * [fit] 重构插件公共类到fit_common_struct * [fit] 删除entity.py中的GenericableInfo和FitableInfo，保留core中的Genericable和Fitable定义 * [fit] 格式修改 * [fit] 格式修改 * [fit] 代码重构 * [fit] 连接模式修改 * [fit] 增加entity.py中类的方法、修改默认注册中心地址 * [fit] 修改小错误 * [fit] 修改小错误 * [fit] 增加线程池清理函数 * [fit] 移动部分配置到插件内部
1 parent 04d617f commit d31f77c

File tree

27 files changed

+1402
-127
lines changed

27 files changed

+1402
-127
lines changed

‎framework/fit/python/bootstrap/fit_py_service_db/service_db.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ def register_all_fit_services() -> None:
127127
fitable_aliases_infos = reduce(list.__add__, list(_plugin_fitable_dict.values()))
128128
local_fitable_aliases_infos = []
129129
for fitable_aliases_info in fitable_aliases_infos:
130-
if not _local_only_invoke(fitable_aliases_info.fitable.genericable_id):
130+
if not _local_only_invoke(fitable_aliases_info.fitable.genericableId):
131131
local_fitable_aliases_infos.append(fitable_aliases_info)
132132
online_fit_services(local_fitable_aliases_infos)
133133
except FitException:

‎framework/fit/python/conf/application.yml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,24 @@ debug-console: true
1717
terminate-main:
1818
enabled: false
1919
local_ip: "localhost"
20-
context-path: ""
20+
context-path: ""
21+
http:
22+
server:
23+
enabled: true
24+
address:
25+
use-random-port: false
26+
port: 9666
27+
port-to-register:
28+
protocol: 2
29+
formats:
30+
- 1
31+
- 2
32+
registry-center:
33+
server:
34+
mode: 'DIRECT' # DIRECT 表示直连，直接连接内存注册中心；PROXY 表示代理模式，通过本地代理服务连接 Nacos 注册中心
35+
addresses:
36+
- "localhost:8080"
37+
protocol: 2
38+
formats:
39+
- 1
40+
context-path: ""

‎framework/fit/python/conf/fit.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,11 +155,17 @@ fit.public.genericables.2ac926e6e40245b78b7bdda23bcb727b:
155155
route:
156156
default: "ONLINE_FIT_SERVICE_FITABLE_ID"
157157
fit.public.genericables.modelengine.fit.registry.registry-service.query-running-fitables:
158-
name: "QUERY_FITABLE_METAS_GEN_ID"
158+
name: "query_fitable_metas_gen_id"
159159
tags:
160160
- "nonTraceable"
161161
route:
162162
default: "query-running-fitables"
163+
fit.public.genericables.modelengine.fit.registry.registry-service.register-fitables:
164+
name: 'register_fitables_gen_id'
165+
tags:
166+
- 'nonTraceable'
167+
route:
168+
default: 'register-fitables'
163169
fit.public.genericables.GET_FITABLES_OF_GENERICABLE_GEN_ID:
164170
name: "get_fitables_of_genericable"
165171
tags:
@@ -499,4 +505,4 @@ fit.public.genericables.modelengine.fit.get.earliest.start.time:
499505
default: "local-worker"
500506
tags:
501507
- "localOnly"
502-
- "nonTraceable"
508+
- "nonTraceable"

‎framework/fit/python/fit_common_struct/core.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010

1111
class Genericable(object):
1212

13-
def __init__(self, genericable_id: str, genericable_version: str):
14-
self.genericable_id = genericable_id
15-
self.genericable_version = genericable_version
13+
def __init__(self, genericableId: str, genericableVersion: str):
14+
self.genericableId = genericableId
15+
self.genericableVersion = genericableVersion
1616

1717
def __eq__(self, other):
1818
if not isinstance(other, self.__class__):
@@ -28,11 +28,11 @@ def __repr__(self):
2828

2929
class Fitable(object):
3030

31-
def __init__(self, genericable_id: str, genericable_version: str, fitable_id: str, fitable_version: str):
32-
self.genericable_id = genericable_id
33-
self.genericable_version = genericable_version
34-
self.fitable_id = fitable_id
35-
self.fitable_version = fitable_version
31+
def __init__(self, genericableId: str, genericableVersion: str, fitableId: str, fitableVersion: str):
32+
self.genericableId = genericableId
33+
self.genericableVersion = genericableVersion
34+
self.fitableId = fitableId
35+
self.fitableVersion = fitableVersion
3636

3737
def __eq__(self, other):
3838
if not isinstance(other, self.__class__):

framework/fit/python/plugin/fit_py_registry_client/entity.py renamed to framework/fit/python/fit_common_struct/entity.py

Lines changed: 55 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -10,49 +10,25 @@
1010
from numpy import int32
1111

1212
from fit_common_struct.core import Address as AddressInner
13+
from fit_common_struct.core import Fitable
1314

1415

15-
class FitableInfo(object):
16-
17-
def __init__(self, genericableId: str, genericableVersion: str, fitableId: str, fitableVersion: str):
18-
self.genericableId = genericableId
19-
self.genericableVersion = genericableVersion
20-
self.fitableId = fitableId
21-
self.fitableVersion = fitableVersion
22-
23-
def __eq__(self, other):
24-
if not isinstance(other, self.__class__):
25-
return False
26-
return self.__dict__ == other.__dict__
27-
28-
def __hash__(self):
29-
return hash(tuple(self.__dict__.values()))
30-
31-
def __repr__(self):
32-
return str(tuple(self.__dict__.values()))
33-
34-
35-
class GenericableInfo:
36-
37-
def __init__(self, genericableId: str, genericableVersion: str):
38-
self.genericableId = genericableId
39-
self.genericableVersion = genericableVersion
40-
41-
def __eq__(self, other):
42-
if not isinstance(other, self.__class__):
43-
return False
44-
return self.__dict__ == other.__dict__
45-
46-
def __hash__(self):
47-
return hash(tuple(self.__dict__.values()))
48-
49-
def __repr__(self):
50-
return str(tuple(self.__dict__.values()))
16+
def safe_hash_dict(obj_dict):
17+
"""安全地计算包含列表的字典的哈希值"""
18+
hashable_values = []
19+
for value in obj_dict.values():
20+
if isinstance(value, list):
21+
hashable_values.append(tuple(value))
22+
elif isinstance(value, dict):
23+
hashable_values.append(tuple(sorted(value.items())))
24+
else:
25+
hashable_values.append(value)
26+
return hash(tuple(hashable_values))
5127

5228

5329
class FitableMeta(object):
5430

55-
def __init__(self, fitable: FitableInfo, aliases: List[str], formats: List[int32]):
31+
def __init__(self, fitable: Fitable, aliases: List[str], formats: List[int32]):
5632
self.fitable = fitable
5733
self.aliases = aliases
5834

@@ -68,7 +44,8 @@ def __eq__(self, other):
6844
return self.__dict__ == other.__dict__
6945

7046
def __hash__(self):
71-
return hash(tuple(self.__dict__.values()))
47+
# 使用安全的哈希函数处理包含列表的对象
48+
return safe_hash_dict(self.__dict__)
7249

7350
def __repr__(self):
7451
return str(tuple(self.__dict__.values()))
@@ -131,7 +108,7 @@ def __eq__(self, other):
131108
return self.__dict__ == other.__dict__
132109

133110
def __hash__(self):
134-
return hash(tuple(self.__dict__.values()))
111+
return safe_hash_dict(self.__dict__)
135112

136113
def __repr__(self):
137114
return str(tuple(self.__dict__.values()))
@@ -154,7 +131,7 @@ def __eq__(self, other):
154131
return self.__dict__ == other.__dict__
155132

156133
def __hash__(self):
157-
return hash(tuple(self.__dict__.values()))
134+
return safe_hash_dict(self.__dict__)
158135

159136
def __repr__(self):
160137
return str(tuple(self.__dict__.values()))
@@ -178,15 +155,15 @@ def __eq__(self, other):
178155
return self.__dict__ == other.__dict__
179156

180157
def __hash__(self):
181-
return hash(tuple(self.__dict__.values()))
158+
return safe_hash_dict(self.__dict__)
182159

183160
def __repr__(self):
184161
return str(tuple(self.__dict__.values()))
185162

186163

187164
class FitableAddressInstance(object):
188165

189-
def __init__(self, applicationInstances: List[ApplicationInstance], fitable: FitableInfo):
166+
def __init__(self, applicationInstances: List[ApplicationInstance], fitable: Fitable):
190167
self.applicationInstances = applicationInstances
191168
self.fitable = fitable
192169

@@ -196,7 +173,7 @@ def __eq__(self, other):
196173
return self.__dict__ == other.__dict__
197174

198175
def __hash__(self):
199-
return hash(tuple(self.__dict__.values()))
176+
return safe_hash_dict(self.__dict__)
200177

201178
def __repr__(self):
202179
return str(tuple(self.__dict__.values()))
@@ -214,7 +191,41 @@ def __eq__(self, other):
214191
return self.__dict__ == other.__dict__
215192

216193
def __hash__(self):
217-
return hash(tuple(self.__dict__.values()))
194+
return safe_hash_dict(self.__dict__)
195+
196+
def __repr__(self):
197+
return str(tuple(self.__dict__.values()))
198+
199+
class HeartBeatInfo(object):
200+
201+
def __init__(self, sceneType: str, aliveTime: int, initDelay: int):
202+
self.sceneType: str = sceneType
203+
self.aliveTime: int = aliveTime
204+
self.initDelay: int = initDelay
205+
206+
def __eq__(self, other):
207+
if not isinstance(other, self.__class__):
208+
return False
209+
return self.__dict__ == other.__dict__
210+
211+
def __hash__(self):
212+
return safe_hash_dict(self.__dict__)
213+
214+
def __repr__(self):
215+
return str(tuple(self.__dict__.values()))
216+
217+
218+
class HeartBeatAddress(object):
219+
def __init__(self, id_: str):
220+
self.id = id_
221+
222+
def __eq__(self, other):
223+
if not isinstance(other, self.__class__):
224+
return False
225+
return self.__dict__ == other.__dict__
226+
227+
def __hash__(self):
228+
return safe_hash_dict(self.__dict__)
218229

219230
def __repr__(self):
220231
return str(tuple(self.__dict__.values()))

‎framework/fit/python/fitframework/const.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,12 +231,23 @@
231231

232232
# registry server
233233
QUERY_FIT_SERVICE_GEN_ID = 'modelengine.fit.registry.registry-service.query-fitables-addresses'
234+
QUERY_FIT_SERVICE_FIT_ID = 'query-fitables-addresses'
234235
SUBSCRIBE_FIT_SERVICE_GEN_ID = 'modelengine.fit.registry.registry-service.subscribe-fitables'
236+
SUBSCRIBE_FIT_SERVICE_FIT_ID = 'subscribe-fitables'
237+
UNSUBSCRIBE_FIT_SERVICE_GEN_ID = 'modelengine.fit.registry.registry-service.unsubscribe-fitables'
238+
UNSUBSCRIBE_FIT_SERVICE_FIT_ID = 'unsubscribe-fitables'
235239
REGISTER_FIT_SERVICE_GEN_ID = 'modelengine.fit.registry.registry-service.register-fitables'
240+
REGISTER_FIT_SERVICE_FIT_ID = 'register-fitables'
241+
UNREGISTER_FIT_SERVICE_GEN_ID = 'modelengine.fit.registry.registry-service.unregister-fitables'
242+
UNREGISTER_FIT_SERVICE_FIT_ID = 'unregister-fitables'
236243
QUERY_FITABLE_METAS_GEN_ID = 'modelengine.fit.registry.registry-service.query-running-fitables'
244+
QUERY_FITABLE_METAS_FIT_ID = 'query-running-fitables'
237245

238246
# heartbeat server
239-
HEART_BEAT_GEN_ID = 'modelengine.fit.heartbeat.send-heartbeat'
247+
SEND_HEART_BEAT_GEN_ID = 'modelengine.fit.heartbeat.send-heartbeat'
248+
SEND_HEART_BEAT_FIT_ID = 'send-heartbeat'
249+
STOP_HEART_BEAT_GEN_ID = 'modelengine.fit.heartbeat.stop-heartbeat'
250+
STOP_HEART_BEAT_FIT_ID = 'stop-heartbeat'
240251

241252
# debugger
242253
DEBUGGER_START_FIT_ID = 'debugger_start_fitable_id'

‎framework/fit/python/fitframework/core/broker/configure_based_brokerimpl.py‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ def default_load_balancing(self, generic_id, fitable_id, fitable: Fitable):
317317
f"addresses count: {len(addresses)}")
318318
if len(addresses) == 0:
319319
fit_logger.warning(f"cannot get any address can use in this worker. "
320-
f"[genericable_id={fitable.genericable_id}, fitable_id={fitable.fitable_id}]")
320+
f"[genericable_id={fitable.genericableId}, fitable_id={fitable.fitableId}]")
321321
return None
322322
# no choice!
323323
if len(addresses) == 1:
@@ -339,13 +339,13 @@ def get_fit_service_addresses(self, fitable: Fitable) -> List[Address]:
339339
addresses: List[Address] = _get_fit_service_address_with_priorities(fitable)
340340
if not addresses:
341341
fit_logger.warning(f"cannot get any endpoint after checking format and protocol. "
342-
f"[genericable_id={fitable.genericable_id}, fitable_id={fitable.fitable_id}]")
342+
f"[genericable_id={fitable.genericableId}, fitable_id={fitable.fitableId}]")
343343
return []
344344

345345
addresses: List[Address] = _load_balance_env_filtering(addresses)
346346
if not addresses:
347347
fit_logger.warning(f"cannot get any endpoint after filtering by environment. "
348-
f"[genericable_id={fitable.genericable_id}, fitable_id={fitable.fitable_id}]")
348+
f"[genericable_id={fitable.genericableId}, fitable_id={fitable.fitableId}]")
349349
return []
350350

351351
return addresses
@@ -398,7 +398,7 @@ def custom_load_balancing(self, address_filter: Callable[[Address], bool],
398398
addresses: List[Address] = self.get_fit_service_addresses(fitable)
399399
if len(addresses) == 0:
400400
fit_logger.warning(f"cannot get any address can use in this worker. "
401-
f"[genericable_id={fitable.genericable_id}, fitable_id={fitable.fitable_id}]")
401+
f"[genericable_id={fitable.genericableId}, fitable_id={fitable.fitableId}]")
402402
return None
403403
try:
404404
addresses = [address for address in addresses if address_filter(address)]
@@ -407,15 +407,15 @@ def custom_load_balancing(self, address_filter: Callable[[Address], bool],
407407
return None
408408
if not addresses:
409409
fit_logger.warning(f"cannot get any address after custom load balancing. "
410-
f"[genericable_id={fitable.genericable_id}, fitable_id={fitable.fitable_id}]")
410+
f"[genericable_id={fitable.genericableId}, fitable_id={fitable.fitableId}]")
411411
return None
412412
if len(addresses) > 1:
413413
fit_logger.warning(f"get more than one address after custom load balancing. "
414-
f"[genericable_id={fitable.genericable_id}, fitable_id={fitable.fitable_id}]")
414+
f"[genericable_id={fitable.genericableId}, fitable_id={fitable.fitableId}]")
415415
return addresses[0]
416416

417417
if addresses[0].id == _worker_id():
418-
return service_repo.get_fitable_ref(fitable.genericable_id, fitable.fitable_id)
418+
return service_repo.get_fitable_ref(fitable.genericableId, fitable.fitableId)
419419

420420
return addresses[0]
421421

@@ -466,9 +466,9 @@ def lb_call_template(fitable_info: Fitable, target_addresses: List[Address]) ->
466466
pass
467467

468468
args = fitable, addresses
469-
lb_fitable_id = get_fit_ffp_fitable_id(fitable.genericable_id, 'load_balance')
469+
lb_fitable_id = get_fit_ffp_fitable_id(fitable.genericableId, 'load_balance')
470470
if lb_fitable_id:
471-
fit_invoke_info = (fitable.genericable_id, lb_fitable_id, lb_call_template)
471+
fit_invoke_info = (fitable.genericableId, lb_fitable_id, lb_call_template)
472472
return _ffp_invoke(fit_invoke_info, False, None, None, *args)
473473
else:
474474
fit_invoke_info = (const.LOAD_BALANCING_GEN_ID, const.LOAD_BALANCING_RANDOM_FIT_ID, lb_call_template)
@@ -591,7 +591,7 @@ def _get_fit_service_address_with_priorities(fitable: Fitable) -> List[Address]:
591591

592592
def _get_fit_service_address_and_convert(fitable: Fitable) -> List[Address]:
593593
addresses: List[Address] = get_fit_service_address_list(fitable)
594-
fit_logger.debug(f"got address, gid: {fitable.genericable_id}, count: {len(addresses)}")
594+
fit_logger.debug(f"got address, gid: {fitable.genericableId}, count: {len(addresses)}")
595595
return addresses
596596

597597

‎framework/fit/python/plugin/fit_py_heart_beat_agent/heart_beat_agent.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from fitframework.api.decorators import register_event
2727
from fitframework.api.enums import FrameworkEvent as Fit_Event
2828
from fitframework.api.logging import sys_plugin_logger
29-
from .heart_beat_utils import HeartBeatAddress, HeartBeatInfo
29+
from fit_common_struct.entity import HeartBeatInfo, HeartBeatAddress
3030

3131
# 用于控制心跳任务退出的队列
3232
_HEART_BEAT_FINISH_QUEUE = multiprocessing.Queue()
@@ -71,7 +71,7 @@ def get_runtime_worker_id() -> str:
7171
pass
7272

7373

74-
@fit(const.HEART_BEAT_GEN_ID)
74+
@fit(const.SEND_HEART_BEAT_GEN_ID)
7575
def heartbeat(beat_info: List[HeartBeatInfo], address: HeartBeatAddress) -> bool:
7676
""" 可能返回 false，也可能抛出异常，也可能超时 """
7777
pass

‎framework/fit/python/plugin/fit_py_heart_beat_agent/heart_beat_utils.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,6 @@
1212
from fitframework.api.logging import sys_plugin_logger
1313

1414

15-
class HeartBeatInfo:
16-
def __init__(self, sceneType: str, aliveTime: int, initDelay: int):
17-
self.sceneType: str = sceneType
18-
self.aliveTime: int = aliveTime
19-
self.initDelay: int = initDelay
20-
21-
22-
class HeartBeatAddress:
23-
def __init__(self, id_: str):
24-
self.id = id_
25-
26-
2715
def timeout_or_exception_retry(timeout: int = 3, a_exception=Exception, max_retry: int = 1):
2816
"""
2917

‎framework/fit/python/plugin/fit_py_http_client/conf/application.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ https:
88
key_path: "plugin/fit_py_http_client/resources/global.key"
99
key_file_encrypted: false # 私钥是否被加密，仅当 cert_enabled 为 true 时有意义
1010
key_file_password: "" # 私钥的密码，仅当 cert_enabled 和 key_file_encrypted 为 true 时有意义
11-
key_file_password_encrypted: false # 私钥的密码是否被加密，仅当 cert_enabled 和 key_file_encrypted 为 true 时有意义
11+
key_file_password_encrypted: false # 私钥的密码是否被加密，仅当 cert_enabled 和 key_file_encrypted 为 true 时有意义

0 commit comments

Comments
 (0)