Refactor to use get_field_value() for safer field access to avoid KeyError exceptions

Author: cyberflying

infra/scripts/index_scripts/03_cu_process_data_text.py

@@ -195,6 +195,10 @@ def create_tables():
 
 create_tables()
 
+def get_field_value(fields, field_name, default=""):
+    field = fields.get(field_name, {})
+    return field.get('valueString', default)
+
 # Process files and insert into DB and Search
 conversationIds, docs, counter = [], [], 0
 for path in paths:
@@ -210,17 +214,17 @@ def create_tables():
         start_timestamp = datetime.strptime(start_time, timestamp_format)
         conversation_id = file_name.split('convo_', 1)[1].split('_')[0]
         conversationIds.append(conversation_id)
-        duration = int(result['result']['contents'][0]['fields']['Duration']['valueString'])
+        fields = result['result']['contents'][0]['fields']
+        duration = int(get_field_value(fields, 'Duration', '0'))
         end_timestamp = str(start_timestamp + timedelta(seconds=duration)).split(".")[0]
         start_timestamp = str(start_timestamp).split(".")[0]
-        fields = result['result']['contents'][0]['fields']
-        summary = fields['summary']['valueString']
-        satisfied = fields['satisfied']['valueString']
-        sentiment = fields['sentiment']['valueString']
-        topic = fields['topic']['valueString']
-        key_phrases = fields['keyPhrases']['valueString']
-        complaint = fields['complaint']['valueString']
-        content = fields['content']['valueString']
+        summary = get_field_value(fields, 'summary')
+        satisfied = get_field_value(fields, 'satisfied')
+        sentiment = get_field_value(fields, 'sentiment')
+        topic = get_field_value(fields, 'topic')
+        key_phrases = get_field_value(fields, 'keyPhrases')
+        complaint = get_field_value(fields, 'complaint')
+        content = get_field_value(fields, 'content')
         cursor.execute(
             "INSERT INTO processed_data (ConversationId, EndTime, StartTime, Content, summary, satisfied, sentiment, topic, key_phrases, complaint) VALUES (?,?,?,?,?,?,?,?,?,?)",
             (conversation_id, end_timestamp, start_timestamp, content, summary, satisfied, sentiment, topic, key_phrases, complaint)

infra/scripts/index_scripts/03_cu_process_data_text_manual.py

@@ -198,6 +198,10 @@ def create_tables():
 
 create_tables()
 
+def get_field_value(fields, field_name, default=""):
+    field = fields.get(field_name, {})
+    return field.get('valueString', default)
+
 # Process files and insert into DB and Search
 conversationIds, docs, counter = [], [], 0
 for path in paths:
@@ -213,17 +217,17 @@ def create_tables():
         start_timestamp = datetime.strptime(start_time, timestamp_format)
         conversation_id = file_name.split('convo_', 1)[1].split('_')[0]
         conversationIds.append(conversation_id)
-        duration = int(result['result']['contents'][0]['fields']['Duration']['valueString'])
+        fields = result['result']['contents'][0]['fields']
+        duration = int(get_field_value(fields, 'Duration', '0'))
         end_timestamp = str(start_timestamp + timedelta(seconds=duration)).split(".")[0]
         start_timestamp = str(start_timestamp).split(".")[0]
-        fields = result['result']['contents'][0]['fields']
-        summary = fields['summary']['valueString']
-        satisfied = fields['satisfied']['valueString']
-        sentiment = fields['sentiment']['valueString']
-        topic = fields['topic']['valueString']
-        key_phrases = fields['keyPhrases']['valueString']
-        complaint = fields['complaint']['valueString']
-        content = fields['content']['valueString']
+        summary = get_field_value(fields, 'summary')
+        satisfied = get_field_value(fields, 'satisfied')
+        sentiment = get_field_value(fields, 'sentiment')
+        topic = get_field_value(fields, 'topic')
+        key_phrases = get_field_value(fields, 'keyPhrases')
+        complaint = get_field_value(fields, 'complaint')
+        content = get_field_value(fields, 'content')
         cursor.execute(
             "INSERT INTO processed_data (ConversationId, EndTime, StartTime, Content, summary, satisfied, sentiment, topic, key_phrases, complaint) VALUES (?,?,?,?,?,?,?,?,?,?)",
             (conversation_id, end_timestamp, start_timestamp, content, summary, satisfied, sentiment, topic, key_phrases, complaint)

infra/scripts/index_scripts/04_cu_process_data_new_data.py

@@ -266,6 +266,10 @@ def create_tables():
 
 create_tables()
 
+def get_field_value(fields, field_name, default=""):
+    field = fields.get(field_name, {})
+    return field.get('valueString', default)
+
 ANALYZER_ID = "ckm-json"
 # Process files and insert into DB and Search - transcripts
 conversationIds, docs, counter = [], [], 0
@@ -282,17 +286,17 @@ def create_tables():
         start_timestamp = datetime.strptime(start_time, timestamp_format)
         conversation_id = file_name.split('convo_', 1)[1].split('_')[0]
         conversationIds.append(conversation_id)
-        duration = int(result['result']['contents'][0]['fields']['Duration']['valueString'])
+        fields = result['result']['contents'][0]['fields']
+        duration = int(get_field_value(fields, 'Duration', '0'))
         end_timestamp = str(start_timestamp + timedelta(seconds=duration)).split(".")[0]
         start_timestamp = str(start_timestamp).split(".")[0]
-        fields = result['result']['contents'][0]['fields']
-        summary = fields['summary']['valueString']
-        satisfied = fields['satisfied']['valueString']
-        sentiment = fields['sentiment']['valueString']
-        topic = fields['topic']['valueString']
-        key_phrases = fields['keyPhrases']['valueString']
-        complaint = fields['complaint']['valueString']
-        content = fields['content']['valueString']
+        summary = get_field_value(fields, 'summary')
+        satisfied = get_field_value(fields, 'satisfied')
+        sentiment = get_field_value(fields, 'sentiment')
+        topic = get_field_value(fields, 'topic')
+        key_phrases = get_field_value(fields, 'keyPhrases')
+        complaint = get_field_value(fields, 'complaint')
+        content = get_field_value(fields, 'content')
         cursor.execute(
             "INSERT INTO processed_data (ConversationId, EndTime, StartTime, Content, summary, satisfied, sentiment, topic, key_phrases, complaint) VALUES (?,?,?,?,?,?,?,?,?,?)",
             (conversation_id, end_timestamp, start_timestamp, content, summary, satisfied, sentiment, topic, key_phrases, complaint)
@@ -339,19 +343,20 @@ def create_tables():
         conversation_id = file_name.split('convo_', 1)[1].split('_')[0]
         conversationIds.append(conversation_id)
 
-        duration = int(result['result']['contents'][0]['fields']['Duration']['valueString'])
+        fields = result['result']['contents'][0]['fields']
+        duration = int(get_field_value(fields, 'Duration', '0'))
         end_timestamp = str(start_timestamp + timedelta(seconds=duration))
         end_timestamp = end_timestamp.split(".")[0]
         start_timestamp = str(start_timestamp).split(".")[0]
 
-        summary = result['result']['contents'][0]['fields']['summary']['valueString']
-        satisfied = result['result']['contents'][0]['fields']['satisfied']['valueString']
-        sentiment = result['result']['contents'][0]['fields']['sentiment']['valueString']
-        topic = result['result']['contents'][0]['fields']['topic']['valueString']
-        key_phrases = result['result']['contents'][0]['fields']['keyPhrases']['valueString']
-        complaint = result['result']['contents'][0]['fields']['complaint']['valueString']
-        content = result['result']['contents'][0]['fields']['content']['valueString']
-        # print(topic)
+        summary = get_field_value(fields, 'summary')
+        satisfied = get_field_value(fields, 'satisfied')
+        sentiment = get_field_value(fields, 'sentiment')
+        topic = get_field_value(fields, 'topic')
+        key_phrases = get_field_value(fields, 'keyPhrases')
+        complaint = get_field_value(fields, 'complaint')
+        content = get_field_value(fields, 'content')
+
         cursor.execute(f"INSERT INTO processed_data (ConversationId, EndTime, StartTime, Content, summary, satisfied, sentiment, topic, key_phrases, complaint) VALUES (?,?,?,?,?,?,?,?,?,?)", (conversation_id, end_timestamp, start_timestamp, content, summary, satisfied, sentiment, topic, key_phrases, complaint))    
         conn.commit()