Skip to content

Commit c750f8a

Browse files
Utilizes file_interceptor parameter and sets it to get_normalized_path() function. (#36738)
* changes to uses path_normalization with file_interceptor in CONFIG * pylint, pyformat and originally pushed wrong test cases (fixed) * pyformatter / pylint fixes * clean up unused part of cloudpickle_pickler_test
1 parent d7bb223 commit c750f8a

File tree

2 files changed

+28
-3
lines changed

2 files changed

+28
-3
lines changed

sdks/python/apache_beam/internal/cloudpickle_pickler.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,14 @@
3737

3838
from apache_beam.internal import code_object_pickler
3939
from apache_beam.internal.cloudpickle import cloudpickle
40+
from apache_beam.internal.code_object_pickler import get_normalized_path
4041

4142
DEFAULT_CONFIG = cloudpickle.CloudPickleConfig(
42-
skip_reset_dynamic_type_state=True)
43-
NO_DYNAMIC_CLASS_TRACKING_CONFIG = cloudpickle.CloudPickleConfig(
44-
id_generator=None, skip_reset_dynamic_type_state=True)
43+
skip_reset_dynamic_type_state=True,
44+
filepath_interceptor=get_normalized_path)
4545
STABLE_CODE_IDENTIFIER_CONFIG = cloudpickle.CloudPickleConfig(
4646
skip_reset_dynamic_type_state=True,
47+
filepath_interceptor=get_normalized_path,
4748
get_code_object_params=cloudpickle.GetCodeObjectParams(
4849
get_code_object_identifier=code_object_pickler.
4950
get_code_object_identifier,

sdks/python/apache_beam/internal/cloudpickle_pickler_test.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,15 @@
1919

2020
# pytype: skip-file
2121

22+
import os
2223
import threading
2324
import types
2425
import unittest
26+
from unittest import mock
2527

2628
from apache_beam.coders import proto2_coder_test_messages_pb2
29+
from apache_beam.internal import cloudpickle_pickler as beam_cloudpickle
30+
from apache_beam.internal import code_object_pickler
2731
from apache_beam.internal import module_test
2832
from apache_beam.internal.cloudpickle_pickler import dumps
2933
from apache_beam.internal.cloudpickle_pickler import loads
@@ -220,6 +224,26 @@ def test_best_effort_determinism_not_implemented(self):
220224
'Ignoring unsupported option: enable_best_effort_determinism',
221225
'\n'.join(l.output))
222226

227+
@mock.patch.object(
228+
beam_cloudpickle.DEFAULT_CONFIG, 'filepath_interceptor', autospec=True)
229+
def test_default_config_interceptor(self, mock_filepath_interceptor):
230+
"""Tests config.filepath_interceptor is called for CodeType pickling."""
231+
mock_filepath_interceptor.side_effect = (
232+
code_object_pickler.get_normalized_path)
233+
234+
def sample_func():
235+
return "Beam"
236+
237+
code_obj = sample_func.__code__
238+
original_filename = os.path.abspath(code_obj.co_filename)
239+
pickled_code = beam_cloudpickle.dumps(code_obj)
240+
unpickled_code = beam_cloudpickle.loads(pickled_code)
241+
242+
mock_filepath_interceptor.assert_called()
243+
244+
unpickled_filename = os.path.abspath(unpickled_code.co_filename)
245+
self.assertEqual(unpickled_filename, original_filename)
246+
223247

224248
if __name__ == '__main__':
225249
unittest.main()

0 commit comments

Comments
 (0)