Skip to content

Commit dedbdf5

Browse files
committed
Remove schema registry from pickle for now
It is currently internal, and test fails on non-cython Beam due to unpickable proto objects
1 parent 5bfae4e commit dedbdf5

File tree

1 file changed

+6
-12
lines changed

1 file changed

+6
-12
lines changed

sdks/python/apache_beam/internal/cloudpickle_pickler.py

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -197,27 +197,21 @@ def _lock_reducer(obj):
197197

198198
def dump_session(file_path):
199199
# Since References are saved (https://s.apache.org/beam-picklers), we only
200-
# dump known Beam Registries
200+
# dump supported Beam Registries (currently only logical type registry)
201201
from apache_beam.typehints.schemas import LogicalType
202-
from apache_beam.typehints import schema_registry
203202

204203
with _pickle_lock, open(file_path, 'wb') as file:
205204
pickler = cloudpickle.CloudPickler(file)
206205
# Logical Type registry
207206
pickler.dump(LogicalType._known_logical_types)
208-
# Schema registry
209-
pickler.dump(schema_registry.SCHEMA_REGISTRY)
207+
# TODO(yathu) consider pickle schema registry, filesystem registry, etc
208+
# after these get implemented and/or promoted as public API
210209

211210

212211
def load_session(file_path):
213212
# It is possible to load_session with cloudpickle. However, since references
214213
# are saved it should not be necessary. See https://s.apache.org/beam-picklers
215-
with _pickle_lock, open(file_path, 'rb') as file:
216-
from apache_beam.typehints.schemas import LogicalType
217-
from apache_beam.typehints import schema_registry
218-
219-
known_logical_type = cloudpickle.load(file)
220-
schema_registry = cloudpickle.load(file)
214+
from apache_beam.typehints.schemas import LogicalType
221215

222-
LogicalType._known_logical_types = known_logical_type
223-
schema_registry.SCHEMA_REGISTRY = schema_registry
216+
with _pickle_lock, open(file_path, 'rb') as file:
217+
LogicalType._known_logical_types = cloudpickle.load(file)

0 commit comments

Comments
 (0)