Skip to content
This repository was archived by the owner on Sep 17, 2025. It is now read-only.

Commit d6114d7

Browse files
authored
Implement max size logic for local storage for Azure exporters (#876)
1 parent 99821a0 commit d6114d7

File tree

3 files changed

+75
-3
lines changed

3 files changed

+75
-3
lines changed

contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ def __init__(self, *args, **kwargs):
9797
minimum_retry_interval=60, # minimum retry interval in seconds
9898
proxy=None,
9999
storage_maintenance_period=60,
100-
storage_max_size=100*1024*1024,
100+
storage_max_size=50*1024*1024, # 50MiB
101101
storage_path=os.path.join(
102102
os.path.expanduser('~'),
103103
'.opencensus',

contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
import datetime
22
import json
3+
import logging
34
import os
45
import random
56

67
from opencensus.common.schedule import PeriodicTask
78

9+
logger = logging.getLogger(__name__)
10+
811

912
def _fmt(timestamp):
1013
return timestamp.strftime('%Y-%m-%dT%H%M%S.%f')
@@ -77,7 +80,7 @@ class LocalFileStorage(object):
7780
def __init__(
7881
self,
7982
path,
80-
max_size=100*1024*1024, # 100MB
83+
max_size=50*1024*1024, # 50MiB
8184
maintenance_period=60, # 1 minute
8285
retention_period=7*24*60*60, # 7 days
8386
write_timeout=60, # 1 minute
@@ -162,6 +165,8 @@ def get(self):
162165
return None
163166

164167
def put(self, data, lease_period=0, silent=False):
168+
if not self._check_storage_size():
169+
return None
165170
blob = LocalFileBlob(os.path.join(
166171
self.path,
167172
'{}-{}.blob'.format(
@@ -170,3 +175,28 @@ def put(self, data, lease_period=0, silent=False):
170175
),
171176
))
172177
return blob.put(data, lease_period=lease_period, silent=silent)
178+
179+
def _check_storage_size(self):
180+
size = 0
181+
for dirpath, dirnames, filenames in os.walk(self.path):
182+
for f in filenames:
183+
fp = os.path.join(dirpath, f)
184+
# skip if it is symbolic link
185+
if not os.path.islink(fp):
186+
try:
187+
size += os.path.getsize(fp)
188+
except OSError:
189+
logger.error(
190+
"Path %s does not exist or is inaccessible.", fp
191+
)
192+
continue
193+
if size >= self.max_size:
194+
logger.warning(
195+
"Persistent storage max capacity has been "
196+
"reached. Currently at %fKB. Telemetry will be "
197+
"lost. Please consider increasing the value of "
198+
"'storage_max_size' in exporter config.",
199+
format(size/1024)
200+
)
201+
return False
202+
return True

contrib/opencensus-ext-azure/tests/test_storage.py

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
_seconds,
2626
)
2727

28-
TEST_FOLDER = os.path.abspath('.test')
28+
TEST_FOLDER = os.path.abspath('.test.storage')
2929

3030

3131
def setUpModule():
@@ -116,6 +116,48 @@ def test_put(self):
116116
self.assertIsNone(stor.put(input, silent=True))
117117
self.assertRaises(Exception, lambda: stor.put(input))
118118

119+
def test_put_max_size(self):
120+
input = (1, 2, 3)
121+
with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd')) as stor:
122+
size_mock = mock.Mock()
123+
size_mock.return_value = False
124+
stor._check_storage_size = size_mock
125+
stor.put(input)
126+
self.assertEqual(stor.get(), None)
127+
128+
def test_check_storage_size_full(self):
129+
input = (1, 2, 3)
130+
with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd2'), 1) as stor:
131+
stor.put(input)
132+
self.assertFalse(stor._check_storage_size())
133+
134+
def test_check_storage_size_not_full(self):
135+
input = (1, 2, 3)
136+
with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor:
137+
stor.put(input)
138+
self.assertTrue(stor._check_storage_size())
139+
140+
def test_check_storage_size_no_files(self):
141+
with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor:
142+
self.assertTrue(stor._check_storage_size())
143+
144+
def test_check_storage_size_links(self):
145+
input = (1, 2, 3)
146+
with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd4'), 1000) as stor:
147+
stor.put(input)
148+
with mock.patch('os.path.islink') as os_mock:
149+
os_mock.return_value = True
150+
self.assertTrue(stor._check_storage_size())
151+
152+
def test_check_storage_size_error(self):
153+
input = (1, 2, 3)
154+
with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd5'), 1) as stor:
155+
with mock.patch('os.path.getsize', side_effect=throw(OSError)):
156+
stor.put(input)
157+
with mock.patch('os.path.islink') as os_mock:
158+
os_mock.return_value = True
159+
self.assertTrue(stor._check_storage_size())
160+
119161
def test_maintanence_routine(self):
120162
with mock.patch('os.makedirs') as m:
121163
m.return_value = None

0 commit comments

Comments
 (0)