|
22 | 22 | # |
23 | 23 | """Stripped out routines for telemetry""" |
24 | 24 |
|
25 | | -import os |
26 | | -import re |
27 | | - |
28 | 25 | from nibabel.optpkg import optional_package |
29 | | -from niworkflows.utils.misc import read_crashfile |
30 | 26 |
|
31 | 27 | from .. import __version__, config |
32 | 28 |
|
33 | | -sentry_sdk = optional_package('sentry_sdk')[0] |
34 | 29 | migas = optional_package('migas')[0] |
35 | 30 |
|
36 | | -CHUNK_SIZE = 16384 |
37 | | -# Group common events with pre specified fingerprints |
38 | | -KNOWN_ERRORS = { |
39 | | - 'permission-denied': ['PermissionError: [Errno 13] Permission denied'], |
40 | | - 'memory-error': [ |
41 | | - 'MemoryError', |
42 | | - 'Cannot allocate memory', |
43 | | - 'Return code: 134', |
44 | | - ], |
45 | | - 'reconall-already-running': ['ERROR: it appears that recon-all is already running'], |
46 | | - 'no-disk-space': ['[Errno 28] No space left on device', '[Errno 122] Disk quota exceeded'], |
47 | | - 'segfault': [ |
48 | | - 'Segmentation Fault', |
49 | | - 'Segfault', |
50 | | - 'Return code: 139', |
51 | | - ], |
52 | | - 'potential-race-condition': [ |
53 | | - '[Errno 39] Directory not empty', |
54 | | - '_unfinished.json', |
55 | | - ], |
56 | | - 'keyboard-interrupt': [ |
57 | | - 'KeyboardInterrupt', |
58 | | - ], |
59 | | -} |
60 | | - |
61 | | - |
62 | | -def sentry_setup(): |
63 | | - """Set-up sentry.""" |
64 | | - release = config.environment.version or 'dev' |
65 | | - environment = ( |
66 | | - 'dev' |
67 | | - if ( |
68 | | - os.getenv('PETPREP_DEV', '').lower in ('1', 'on', 'yes', 'y', 'true') |
69 | | - or ('+' in release) |
70 | | - ) |
71 | | - else 'prod' |
72 | | - ) |
73 | | - |
74 | | - sentry_sdk.init( |
75 | | - 'https://[email protected]/1137693', |
76 | | - release=release, |
77 | | - environment=environment, |
78 | | - before_send=before_send, |
79 | | - ) |
80 | | - with sentry_sdk.configure_scope() as scope: |
81 | | - for k, v in config.get(flat=True).items(): |
82 | | - scope.set_tag(k, v) |
83 | | - |
84 | | - |
85 | | -def process_crashfile(crashfile): |
86 | | - """Parse the contents of a crashfile and submit sentry messages.""" |
87 | | - crash_info = read_crashfile(str(crashfile)) |
88 | | - with sentry_sdk.push_scope() as scope: |
89 | | - scope.level = 'fatal' |
90 | | - |
91 | | - # Extract node name |
92 | | - node_name = crash_info.pop('node').split('.')[-1] |
93 | | - scope.set_tag('node_name', node_name) |
94 | | - |
95 | | - # Massage the traceback, extract the gist |
96 | | - traceback = crash_info.pop('traceback') |
97 | | - # last line is probably most informative summary |
98 | | - gist = traceback.splitlines()[-1] |
99 | | - exception_text_start = 1 |
100 | | - for line in traceback.splitlines()[1:]: |
101 | | - if not line[0].isspace(): |
102 | | - break |
103 | | - exception_text_start += 1 |
104 | | - |
105 | | - exception_text = '\n'.join(traceback.splitlines()[exception_text_start:]) |
106 | | - |
107 | | - # Extract inputs, if present |
108 | | - inputs = crash_info.pop('inputs', None) |
109 | | - if inputs: |
110 | | - scope.set_extra('inputs', dict(inputs)) |
111 | | - |
112 | | - # Extract any other possible metadata in the crash file |
113 | | - for k, v in crash_info.items(): |
114 | | - strv = _chunks(str(v)) |
115 | | - if len(strv) == 1: |
116 | | - scope.set_extra(k, strv[0]) |
117 | | - else: |
118 | | - for i, chunk in enumerate(strv): |
119 | | - scope.set_extra(f'{k}_{i:02d}', chunk) |
120 | | - |
121 | | - fingerprint = '' |
122 | | - issue_title = f'{node_name}: {gist}' |
123 | | - for new_fingerprint, error_snippets in KNOWN_ERRORS.items(): |
124 | | - for error_snippet in error_snippets: |
125 | | - if error_snippet in traceback: |
126 | | - fingerprint = new_fingerprint |
127 | | - issue_title = new_fingerprint |
128 | | - break |
129 | | - if fingerprint: |
130 | | - break |
131 | | - |
132 | | - message = issue_title + '\n\n' |
133 | | - message += exception_text[-8192:] |
134 | | - if fingerprint: |
135 | | - sentry_sdk.add_breadcrumb(message=fingerprint, level='fatal') |
136 | | - else: |
137 | | - # remove file paths |
138 | | - fingerprint = re.sub(r'(/[^/ ]*)+/?', '', message) |
139 | | - # remove words containing numbers |
140 | | - fingerprint = re.sub(r'([a-zA-Z]*[0-9]+[a-zA-Z]*)+', '', fingerprint) |
141 | | - # adding the return code if it exists |
142 | | - for line in message.splitlines(): |
143 | | - if line.startswith('Return code'): |
144 | | - fingerprint += line |
145 | | - break |
146 | | - |
147 | | - scope.fingerprint = [fingerprint] |
148 | | - sentry_sdk.capture_message(message, 'fatal') |
149 | | - |
150 | | - |
151 | | -def before_send(event, hints): |
152 | | - """Filter log messages about crashed nodes.""" |
153 | | - if 'logentry' in event and 'message' in event['logentry']: |
154 | | - msg = event['logentry']['message'] |
155 | | - if msg.startswith('could not run node:'): |
156 | | - return None |
157 | | - if msg.startswith('Saving crash info to '): |
158 | | - return None |
159 | | - if re.match('Node .+ failed to run on host .+', msg): |
160 | | - return None |
161 | | - |
162 | | - if 'breadcrumbs' in event and isinstance(event['breadcrumbs'], list): |
163 | | - fingerprints_to_propagate = [ |
164 | | - 'no-disk-space', |
165 | | - 'memory-error', |
166 | | - 'permission-denied', |
167 | | - 'keyboard-interrupt', |
168 | | - ] |
169 | | - for bc in event['breadcrumbs']: |
170 | | - msg = bc.get('message', 'empty-msg') |
171 | | - if msg in fingerprints_to_propagate: |
172 | | - event['fingerprint'] = [msg] |
173 | | - break |
174 | | - |
175 | | - return event |
176 | | - |
177 | | - |
178 | | -def _chunks(string, length=CHUNK_SIZE): |
179 | | - """ |
180 | | - Split a string into smaller chunks. |
181 | | -
|
182 | | - >>> list(_chunks('some longer string.', length=3)) |
183 | | - ['som', 'e l', 'ong', 'er ', 'str', 'ing', '.'] |
184 | | -
|
185 | | - """ |
186 | | - return [string[i : i + length] for i in range(0, len(string), length)] |
187 | | - |
188 | 31 |
|
189 | 32 | def setup_migas(init_ping: bool = True, exit_ping: bool = True) -> None: |
190 | 33 | """ |
|
0 commit comments