generic feature column

Author: mtiessler

backend/dendogram_controller.py

@@ -125,7 +125,7 @@ def generate_dendogram_from_csv():
             file_content = file.stream.read().decode('utf-8')
             csv_reader = csv.DictReader(file_content.splitlines())
             for row in csv_reader:
-                extracted_features = row.get("extracted_features_TransFeatEx", "")
+                extracted_features = row.get("extracted_features", "")
                 if extracted_features:
                     features.extend(extracted_features.split(';'))
             logger.info(f"Extracted {len(features)} features from CSV file")

cli-client/scripts/Stage 1 - Feature extraction/review_postprocessing.py

@@ -43,10 +43,10 @@ def parse_and_add_column(csv_folder, json_folder, output_folder):
                 # Concatenate features separated by ';'
                 review_features_map[review_id] = ";".join(filter(None, features))
 
-        # Add the 'extracted_features_TransFeatEx' column
-        df['extracted_features_TransFeatEx'] = df['reviewId'].map(review_features_map)
-        # Filter out rows where extracted_features_TransFeatEx is empty
-        df = df[df['extracted_features_TransFeatEx'].notna() & (df['extracted_features_TransFeatEx'] != '')]
+        # Add the 'extracted_features' column
+        df['extracted_features'] = df['reviewId'].map(review_features_map)
+        # Filter out rows where extracted_features is empty
+        df = df[df['extracted_features'].notna() & (df['extracted_features'] != '')]
         # Save the updated DataFrame to the output folder
         output_path = os.path.join(output_folder, csv_file)
         df.to_csv(output_path, sep=',', index=False)

cli-client/scripts/Stage 1 - Feature extraction/statistics.py

@@ -35,7 +35,7 @@ def process_files(input_folder, output_folder, top_n):
 
         # Extract and flatten features
         all_features = []
-        for cell in df['extracted_features_TransFeatEx'].dropna():
+        for cell in df['extracted_features'].dropna():
             features = cell.split(';')
             all_features.extend(features)
 
@@ -112,7 +112,7 @@ def process_files(input_folder, output_folder, top_n):
                 category_features[category] = []
 
             # Extract and flatten features for this category
-            for cell in group['extracted_features_TransFeatEx'].dropna():
+            for cell in group['extracted_features'].dropna():
                 features = cell.split(';')
                 category_features[category].extend(features)
 

data/Stage 2 - Hierarchical Clustering/input/TransFeatEx/com.discord.csv

@@ -1,4 +1,4 @@
-app_name,app_package,app_categoryId,reviewId,review,reply,userName,score,at,extracted_features_TransFeatEx
+app_name,app_package,app_categoryId,reviewId,review,reply,userName,score,at,extracted_features
 "Discord - Talk, Play, Hang Out",com.discord,COMMUNICATION,00280421-44e5-4026-8374-72b714bfe6ec,"Buggy (eg. notifications just don't work for me), there's no way to quit when you are done, and it uses a lot of the battery ... but great when it works. Edit: They say to contact support, but I did and they haven't gotten back for weeks now.","Make sure you aren't logged in on another client and try setting server notifications to @mentions. If you're still having trouble, change your password and verify your notification settings with this guide: https://support.discord.com/hc/en-us/articles/218892547",Jeremy Walker,3,"Nov 06, 2024",Buggy;UseLot;Edit
 "Discord - Talk, Play, Hang Out",com.discord,COMMUNICATION,2966add5-6b3f-40dc-8057-41fc61c03dba,it's gotten so many bugs overtime and discord doesn't do anything to fix it. I can't access the shop or change pfps without being kicked out of app.,"Try reinstalling the app and restarting your device. If issues persist, see this article: https://support.discord.com/hc/en-us/articles/13148417007767",Broken Lexi,3,"Nov 06, 2024",GetSoManyBug;AccessShop
 "Discord - Talk, Play, Hang Out",com.discord,COMMUNICATION,cc5fe35b-2338-4183-99bc-4b925c68da43,"One Star. LITERALLY, one freaking star. Because you know why? Well when u try to change my email, I can't. It will always be the old email, can u like fix this?","If you're having trouble updating your email, please submit a support ticket here so we can provide assistance: dis.gd/contact",Princess Javelina,1,"Nov 06, 2024",OneStar;LiterallyOneFreakingStar;ChangeEmail