remove commented code

Author: Pavan-Microsoft

infra/scripts/index_scripts/04_cu_process_data_new_data.py

@@ -10,7 +10,6 @@
 
 from datetime import datetime, timedelta
 import time
-import base64
 import pyodbc
 import struct
 
@@ -121,9 +120,6 @@ def chunk_data(text):
 
 search_client = SearchClient(search_endpoint, index_name, search_credential)
 
-# # delete all the documents in the index
-# search_client.delete_documents(search_client.search("*"))
-
 index_client = SearchIndexClient(endpoint=search_endpoint, credential=search_credential)
 
 # Delete the search index
@@ -401,10 +397,6 @@ def prepare_search_doc(content, document_id):
         cursor.execute(f"INSERT INTO processed_data (ConversationId, EndTime, StartTime, Content, summary, satisfied, sentiment, topic, key_phrases, complaint) VALUES (?,?,?,?,?,?,?,?,?,?)", (conversation_id, end_timestamp, start_timestamp, content, summary, satisfied, sentiment, topic, key_phrases, complaint))    
         conn.commit()
 
-        # keyPhrases = key_phrases.split(',')
-        # for keyPhrase in keyPhrases:
-        #     cursor.execute(f"INSERT INTO processed_data_key_phrases (ConversationId, key_phrase, sentiment) VALUES (?,?,?)", (conversation_id, keyPhrase, sentiment))
-
         document_id = conversation_id
 
         result = prepare_search_doc(content, document_id)
@@ -471,16 +463,6 @@ def call_gpt4(topics_str1, client):
         Return the topics and their labels in JSON format.Always add 'topics' node and 'label', 'description' attributes in json.
         Do not return anything else.
         """
-    # Phi-3 model client
-    # response = client.complete(
-    #     messages=[
-    #         # SystemMessage(content=prompt),
-    #         UserMessage(content=topic_prompt),
-    #     ],
-    #     max_tokens = 1000,
-    #     temperature = 0,
-    #     top_p = 1
-    # )
 
     # GPT-4o model client
     response = client.chat.completions.create(
@@ -588,17 +570,6 @@ def get_mined_topic_mapping(input_text, list_of_topics):
                 from a list of topics - {list_of_topics}.
                 ALWAYS only return a topic from list - {list_of_topics}. Do not add any other text.'''
 
-    # Phi-3 model client
-    # response = client.complete(
-    #     messages=[
-    #         # SystemMessage(content=prompt),
-    #         UserMessage(content=prompt),
-    #     ],
-    #     max_tokens = 500,
-    #     temperature = 0,
-    #     top_p = 1
-    # )
-
     # GPT-4o model client
     response = client.chat.completions.create(
                 model=deployment,
@@ -669,28 +640,8 @@ def get_mined_topic_mapping(input_text, list_of_topics):
 # Bulk insert using executemany()
 cursor.executemany(insert_sql, data_list)
 
-# column_names = [i[0] for i in cursor.description]
-# df = pd.DataFrame(rows, columns=column_names)
-# for idx, row in df.iterrows():
-#     cursor.execute(f"INSERT INTO km_processed_data (ConversationId, StartTime, EndTime, Content, summary, satisfied, sentiment, keyphrases, complaint, topic) VALUES (?,?,?,?,?,?,?,?,?,?)", (row['ConversationId'], row['StartTime'], row['EndTime'], row['Content'], row['summary'], row['satisfied'], row['sentiment'], row['keyphrases'], row['complaint'], row['topic']))
 conn.commit()
 
-# update keyphrase table after the data update
-# cursor.execute('DROP TABLE IF EXISTS processed_data_key_phrases')
-# conn.commit()
-# print("Dropped processed_data_key_phrases table")
-
-# create_processed_data_sql = """CREATE TABLE processed_data_key_phrases (
-#                 ConversationId varchar(255),
-#                 key_phrase varchar(500), 
-#                 sentiment varchar(255),
-#                 topic varchar(255), 
-#                 StartTime varchar(255),
-#             );"""
-# cursor.execute(create_processed_data_sql)
-# conn.commit()
-# print('created processed_data_key_phrases table')
-
 sql_stmt = '''select ConversationId, key_phrases, sentiment, mined_topic as topic, StartTime from processed_data'''
 cursor.execute(sql_stmt)
 rows = [tuple(row) for row in cursor.fetchall()]

infra/scripts/process_data_scripts.sh

@@ -9,27 +9,11 @@ requirementFileUrl=${baseUrl}"infra/scripts/index_scripts/requirements.txt"
 
 echo "Script Started"
 
-# Download the create_index and create table python files
-# curl --output "01_create_search_index.py" ${baseUrl}"infra/scripts/index_scripts/01_create_search_index.py"
-# curl --output "02_create_cu_template_text.py" ${baseUrl}"infra/scripts/index_scripts/02_create_cu_template_text.py"
-# curl --output "02_create_cu_template_audio.py" ${baseUrl}"infra/scripts/index_scripts/02_create_cu_template_audio.py"
-# curl --output "03_cu_process_data_text.py" ${baseUrl}"infra/scripts/index_scripts/03_cu_process_data_text.py"
 curl --output "04_cu_process_data_new_data.py" ${baseUrl}"infra/scripts/index_scripts/04_cu_process_data_new_data.py"
 curl --output "content_understanding_client.py" ${baseUrl}"infra/scripts/index_scripts/content_understanding_client.py"
 curl --output "ckm-analyzer_config_text.json" ${baseUrl}"infra/data/ckm-analyzer_config_text.json"
 curl --output "ckm-analyzer_config_audio.json" ${baseUrl}"infra/data/ckm-analyzer_config_audio.json"
 
-# curl --output "sample_processed_data.json" ${baseUrl}"infra/data/sample_processed_data.json"
-# curl --output "sample_processed_data_key_phrases.json" ${baseUrl}"infra/data/sample_processed_data_key_phrases.json"
-# curl --output "sample_search_index_data.json" ${baseUrl}"infra/data/sample_search_index_data.json"
-
-# RUN apt-get update
-# RUN apt-get install python3 python3-dev g++ unixodbc-dev unixodbc libpq-dev
-# apk add python3 python3-dev g++ unixodbc-dev unixodbc libpq-dev
- 
-# # RUN apt-get install python3 python3-dev g++ unixodbc-dev unixodbc libpq-dev
-# pip install pyodbc
-
 ############################################
 echo "Installing system packages..."    
 apk add --no-cache --virtual .build-deps build-base unixodbc-dev
@@ -46,22 +30,8 @@ curl --output "$requirementFile" "$requirementFileUrl"
 
 echo "Download completed"
 
-#Replace key vault name 
-# sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "01_create_search_index.py"
-# sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "01_create_search_index.py"
-# sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "02_create_cu_template_text.py"
-# sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "02_create_cu_template_text.py"
-# sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "02_create_cu_template_audio.py"
-# sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "02_create_cu_template_audio.py"
-# sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "03_cu_process_data_text.py"
-# sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "03_cu_process_data_text.py"
-
 sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "04_cu_process_data_new_data.py"
 
 pip install -r requirements.txt
 
-# python 01_create_search_index.py
-# python 02_create_cu_template_text.py
-# python 02_create_cu_template_audio.py
-# python 03_cu_process_data_text.py
 python 04_cu_process_data_new_data.py