Merge pull request #63 from seanpedrick-case/dev

Integration of llm deduplication into all in one pipeline (optional). Can now output only xlsx file in direct mode. Improved Excel file output formatting.

Author: seanpedrick-case

README.md

@@ -13,7 +13,7 @@ short_description: Create thematic summaries for open text data with LLMs
 
 # Large language model topic modelling
 
-Version: 0.6.0
+Version: 0.7.0
 
 Extract topics and summarise outputs using Large Language Models (LLMs, Gemma 3 4b/GPT-OSS 20b if local (see tools/config.py to modify), Gemini, Azure, or AWS Bedrock models (e.g. Claude, Nova models). The app will query the LLM with batches of responses to produce summary tables, which are then compared iteratively to output a table with the general topics, subtopics, topic sentiment, and a topic summary. Instructions on use can be found in the README.md file. You can try out examples by clicking on one of the example datasets on the main app page, which will show you example outputs from a local model run. API keys for AWS, Azure, and Gemini services can be entered on the settings page (note that Gemini has a free public API).
 

app.py

@@ -1853,6 +1853,10 @@ def deduplicate_topics_llm_wrapper(
         candidate_topics=None,
         azure_endpoint="",
         api_url=None,
+        aws_access_key_textbox="",
+        aws_secret_key_textbox="",
+        aws_region_textbox="",
+        azure_api_key_textbox="",
     ):
         # Ensure custom model_choice is registered in model_name_map
         ensure_model_in_map(model_choice)
@@ -1880,6 +1884,10 @@ def deduplicate_topics_llm_wrapper(
             azure_endpoint,
             OUTPUT_DEBUG_FILES,
             api_url,
+            aws_access_key_textbox,
+            aws_secret_key_textbox,
+            aws_region_textbox,
+            azure_api_key_textbox,
         )
 
     deduplicate_llm_previous_data_btn.click(
@@ -1912,6 +1920,10 @@ def deduplicate_topics_llm_wrapper(
             candidate_topics,
             azure_endpoint_textbox,
             api_url_textbox,
+            aws_access_key_textbox,
+            aws_secret_key_textbox,
+            aws_region_textbox,
+            azure_api_key_textbox,
         ],
         outputs=[
             master_reference_df_state,

cli_topics.py

@@ -25,6 +25,7 @@
     DEDUPLICATION_THRESHOLD,
     DEFAULT_COST_CODE,
     DEFAULT_SAMPLED_SUMMARIES,
+    DIRECT_MODE_S3_UPLOAD_ONLY_XLSX,
     DYNAMODB_USAGE_LOG_HEADERS,
     GEMINI_API_KEY,
     GRADIO_TEMP_DIR,
@@ -1393,7 +1394,6 @@ def main(direct_mode_args={}):
                     in_api_key=args.google_api_key,
                     temperature=args.temperature,
                     model_source=model_source,
-                    bedrock_runtime=None,
                     local_model=None,
                     tokenizer=None,
                     assistant_model=None,
@@ -1409,6 +1409,19 @@ def main(direct_mode_args={}):
                     azure_endpoint=args.azure_endpoint,
                     output_debug_files=str(args.output_debug_files),
                     api_url=args.api_url if args.api_url else API_URL,
+                    aws_access_key_textbox=(
+                        args.aws_access_key if hasattr(args, "aws_access_key") else ""
+                    ),
+                    aws_secret_key_textbox=(
+                        args.aws_secret_key if hasattr(args, "aws_secret_key") else ""
+                    ),
+                    aws_region_textbox=(
+                        args.aws_region if hasattr(args, "aws_region") else ""
+                    ),
+                    azure_api_key_textbox=(
+                        args.azure_api_key if hasattr(args, "azure_api_key") else ""
+                    ),
+                    model_name_map=model_name_map,
                 )
 
             end_time = time.time()
@@ -1944,15 +1957,15 @@ def main(direct_mode_args={}):
 
             # Upload outputs to S3 if enabled
             # Collect all output files from the pipeline
-            all_output_files = []
-            if topic_extraction_output_files:
-                all_output_files.extend(topic_extraction_output_files)
-            if overall_summary_output_files:
-                all_output_files.extend(overall_summary_output_files)
+            all_s3_output_files = []
+            if topic_extraction_output_files and not DIRECT_MODE_S3_UPLOAD_ONLY_XLSX:
+                all_s3_output_files.extend(topic_extraction_output_files)
+            if overall_summary_output_files and not DIRECT_MODE_S3_UPLOAD_ONLY_XLSX:
+                all_s3_output_files.extend(overall_summary_output_files)
             if xlsx_files:
-                all_output_files.extend(xlsx_files)
+                all_s3_output_files.extend(xlsx_files)
             upload_outputs_to_s3_if_enabled(
-                output_files=all_output_files,
+                output_files=all_s3_output_files,
                 base_file_name=file_name,
                 session_hash=session_hash,
             )

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "llm_topic_modelling"
-version = "0.6.0"
+version = "0.7.0"
 description = "Generate thematic summaries from open text in tabular data files with a large language model."
 requires-python = ">=3.10"
 readme = "README.md"