diff --git a/README.md b/README.md new file mode 100644 index 0000000..501e0d2 --- /dev/null +++ b/README.md @@ -0,0 +1,124 @@ +# Robot Data Separator + +This repository contains scripts to separate robots data from larger JSON files to reduce import size. + +## Files + +- **`separate_robots.py`** - Full-featured script with command line arguments +- **`simple_robot_filter.py`** - Simple script for basic usage +- **`select_robots.py`** - Script to select specific robots (configurable) +- **`interactive_robot_selector.py`** - Interactive script to choose robots at runtime +- **`README.md`** - This instruction file + +## 🚀 Quick Start Options + +### Option 1: Simple Filter (All Robots) +```bash +python simple_robot_filter.py +``` +- Just change the filename at the top and run +- Creates `robots_only.json` with ALL robots + +### Option 2: Select Specific Robots (Configurable) +```bash +python select_robots.py +``` +- Edit the script to choose which robots to include/exclude +- More control over selection + +### Option 3: Interactive Selection (Recommended) +```bash +python interactive_robot_selector.py +``` +- **Best option for choosing robots!** +- Shows you all available robots +- Lets you pick exactly which ones you want +- No need to edit code + +## 🎯 How to Choose Specific Robots + +### Method 1: Interactive Script (Easiest) +1. Run: `python interactive_robot_selector.py` +2. Enter your JSON filename +3. See all available robots with counts +4. Choose from the menu: + - Pick individual robots + - Select "ALL robots" + - Use "Custom selection" for multiple robots +5. Choose your output filename +6. Get your filtered data! + +### Method 2: Edit the Configurable Script +In `select_robots.py`, change these lines: + +```python +# Include only specific robots +INCLUDE_ROBOTS = ["Term.", "Robot1", "Robot2"] # Add robot names you want + +# OR exclude specific robots +EXCLUDE_ROBOTS = ["UnwantedRobot"] # Add robots to exclude + +# Choose which method to use +USE_INCLUDE_LIST = True # True for include, False for exclude +``` + +### Method 3: Command Line (Advanced) +```bash +# Include only specific robots +python separate_robots.py your_file.json --include "Term.,Robot1" + +# Exclude specific robots +python separate_robots.py your_file.json --exclude "UnwantedRobot" +``` + +## 📋 What Each Script Does + +| Script | Purpose | Best For | +|--------|---------|----------| +| `simple_robot_filter.py` | Get ALL robots | Quick filtering | +| `select_robots.py` | Pre-configured selection | Repeatable filtering | +| `interactive_robot_selector.py` | Choose at runtime | **Selecting specific robots** | +| `separate_robots.py` | Command line control | Automation/scripts | + +## 🔍 Example: Selecting Only "Term." Robot + +### Using Interactive Script: +1. Run: `python interactive_robot_selector.py` +2. Enter your filename +3. Choose option 1 (if "Term." is first in the list) +4. Or use "Custom selection" and enter the number + +### Using Configurable Script: +```python +# In select_robots.py +INCLUDE_ROBOTS = ["Term."] # Only include Term. robot +USE_INCLUDE_LIST = True +``` + +## 📊 Expected Output + +The scripts will show you: +- Total records loaded +- Available robots and their counts +- Records found after filtering +- Final robot breakdown +- File size reduction percentage + +## 🛠️ Requirements + +- Python 3.6 or higher +- No additional packages required (uses only built-in libraries) + +## 🚨 Troubleshooting + +- **File not found**: Make sure you've updated the filename in the script +- **JSON errors**: Ensure your file contains valid JSON +- **No robots found**: Check that your "Robô" field contains data +- **Interactive script not working**: Make sure you're running it in a terminal that supports input + +## 💡 Tips + +1. **Start with the interactive script** - it's the easiest way to see what robots you have +2. **Use the configurable script** if you need to run the same filter repeatedly +3. **Check the file size reduction** to see how much data you're saving +4. **Backup your original file** before running any scripts \ No newline at end of file diff --git a/interactive_robot_selector.py b/interactive_robot_selector.py new file mode 100644 index 0000000..3b703a3 --- /dev/null +++ b/interactive_robot_selector.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +Interactive script to select robots from JSON file at runtime. +This script will show you all available robots and let you choose which ones to include. +""" + +import json +import sys +from pathlib import Path + +def load_json_file(filename): + """Load JSON data from file.""" + try: + with open(filename, 'r', encoding='utf-8') as file: + return json.load(file) + except FileNotFoundError: + print(f"❌ Error: File '{filename}' not found.") + return None + except json.JSONDecodeError as e: + print(f"❌ Error: Invalid JSON in file '{filename}': {e}") + return None + +def get_available_robots(data): + """Get all available robots and their counts.""" + robots = {} + for record in data: + if "Robô" in record and record["Robô"]: + robot_name = str(record["Robô"]).strip() + robots[robot_name] = robots.get(robot_name, 0) + 1 + return robots + +def display_robot_menu(robots): + """Display robots in a numbered menu.""" + print("\n🤖 Available robots in your data:") + print("=" * 50) + + robot_list = sorted(robots.items()) + for i, (robot, count) in enumerate(robot_list, 1): + print(f" {i:2d}. {robot:<20} ({count:4d} records)") + + print(f" {len(robot_list) + 1:2d}. ALL robots") + print(f" {len(robot_list) + 2:2d}. Custom selection") + print(f" {len(robot_list) + 3:2d}. Exit") + + return robot_list + +def get_user_selection(robot_list): + """Get user's robot selection.""" + while True: + try: + choice = input(f"\n👉 Enter your choice (1-{len(robot_list) + 3}): ").strip() + choice_num = int(choice) + + if choice_num == len(robot_list) + 1: # ALL robots + return "ALL" + elif choice_num == len(robot_list) + 2: # Custom selection + return get_custom_selection(robot_list) + elif choice_num == len(robot_list) + 3: # Exit + return "EXIT" + elif 1 <= choice_num <= len(robot_list): + selected_robot = robot_list[choice_num - 1][0] + return [selected_robot] + else: + print("❌ Invalid choice. Please try again.") + except ValueError: + print("❌ Please enter a valid number.") + +def get_custom_selection(robot_list): + """Get custom robot selection from user.""" + print(f"\n📝 Enter robot numbers separated by commas (e.g., 1,3,5)") + print(" Or enter 'all' for all robots") + + while True: + choice = input("👉 Your selection: ").strip().lower() + + if choice == "all": + return "ALL" + + try: + numbers = [int(x.strip()) for x in choice.split(",")] + selected_robots = [] + + for num in numbers: + if 1 <= num <= len(robot_list): + selected_robots.append(robot_list[num - 1][0]) + else: + print(f"❌ Invalid number: {num}") + return None + + if selected_robots: + return selected_robots + else: + print("❌ No valid robots selected. Please try again.") + except ValueError: + print("❌ Invalid input. Please enter numbers separated by commas.") + +def filter_data_by_robots(data, selected_robots): + """Filter data based on selected robots.""" + if selected_robots == "ALL": + return [record for record in data if "Robô" in record and record["Robô"]] + + filtered_data = [] + for record in data: + if "Robô" in record and record["Robô"]: + robot_name = str(record["Robô"]).strip() + if robot_name in selected_robots: + filtered_data.append(record) + + return filtered_data + +def save_filtered_data(data, output_filename): + """Save filtered data to file.""" + try: + with open(output_filename, 'w', encoding='utf-8') as file: + json.dump(data, file, indent=2, ensure_ascii=False) + return True + except Exception as e: + print(f"❌ Error saving file: {e}") + return False + +def main(): + """Main interactive function.""" + print("🚀 Interactive Robot Selector") + print("=" * 40) + + # Get input filename + while True: + filename = input("📁 Enter your JSON filename: ").strip() + if filename: + break + print("❌ Please enter a filename.") + + # Load data + print(f"\n📂 Loading {filename}...") + data = load_json_file(filename) + if data is None: + return + + print(f"✅ Loaded {len(data)} total records") + + # Get available robots + robots = get_available_robots(data) + if not robots: + print("❌ No robots found in the data!") + return + + # Display menu and get selection + robot_list = display_robot_menu(robots) + selection = get_user_selection(robot_list) + + if selection == "EXIT": + print("👋 Goodbye!") + return + + # Filter data + print(f"\n🔍 Filtering data...") + if selection == "ALL": + print("📋 Including ALL robots") + else: + print(f"📋 Including: {', '.join(selection)}") + + filtered_data = filter_data_by_robots(data, selection) + print(f"✅ Found {len(filtered_data)} matching records") + + # Get output filename + output_filename = input(f"\n💾 Enter output filename (default: selected_robots.json): ").strip() + if not output_filename: + output_filename = "selected_robots.json" + + # Save data + print(f"\n💾 Saving to {output_filename}...") + if save_filtered_data(filtered_data, output_filename): + print("✅ Data saved successfully!") + + # Show final statistics + final_robots = {} + for record in filtered_data: + robot_name = str(record["Robô"]).strip() + final_robots[robot_name] = final_robots.get(robot_name, 0) + 1 + + print(f"\n📈 Final robot breakdown:") + print("-" * 30) + for robot, count in sorted(final_robots.items()): + print(f" {robot}: {count} records") + + # Calculate size reduction + original_size = Path(filename).stat().st_size + filtered_size = Path(output_filename).stat().st_size + reduction = ((original_size - filtered_size) / original_size) * 100 + + print(f"\n📊 File size reduction:") + print(f" Original: {original_size:,} bytes") + print(f" Filtered: {filtered_size:,} bytes") + print(f" Reduction: {reduction:.1f}%") + + print("\n🎉 Done!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/select_robots.py b/select_robots.py new file mode 100644 index 0000000..49899e4 --- /dev/null +++ b/select_robots.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +""" +Script to select specific robots from JSON file and save them to a separate file. +You can choose which robots to include or exclude. +""" + +import json +import sys +from pathlib import Path + +# CHANGE THESE SETTINGS +INPUT_FILE = "your_file.json" # Replace with your actual filename +OUTPUT_FILE = "selected_robots.json" + +# OPTION 1: Include only specific robots (leave empty list to include all) +INCLUDE_ROBOTS = ["Term.", "Robot1", "Robot2"] # Add robot names you want + +# OPTION 2: Exclude specific robots (leave empty list to exclude none) +EXCLUDE_ROBOTS = [] # Add robot names you want to exclude + +# OPTION 3: Use include list (True) or exclude list (False) +USE_INCLUDE_LIST = True # Set to False to use exclude list instead + +def filter_by_robot_selection(data): + """Filter data based on robot selection criteria.""" + filtered_data = [] + + for record in data: + if "Robô" not in record or not record["Robô"]: + continue + + robot_name = str(record["Robô"]).strip() + + if USE_INCLUDE_LIST: + # Include only specified robots + if INCLUDE_ROBOTS and robot_name in INCLUDE_ROBOTS: + filtered_data.append(record) + elif not INCLUDE_ROBOTS: # If list is empty, include all + filtered_data.append(record) + else: + # Exclude specified robots + if robot_name not in EXCLUDE_ROBOTS: + filtered_data.append(record) + + return filtered_data + +def show_available_robots(data): + """Show all available robots in the data.""" + robots = {} + for record in data: + if "Robô" in record and record["Robô"]: + robot_name = str(record["Robô"]).strip() + robots[robot_name] = robots.get(robot_name, 0) + 1 + + print("\n📊 Available robots in your data:") + print("-" * 40) + for robot, count in sorted(robots.items()): + print(f" {robot}: {count} records") + + return robots + +def main(): + """Main function to process the JSON file.""" + try: + # Load the JSON file + print(f"📁 Loading {INPUT_FILE}...") + with open(INPUT_FILE, 'r', encoding='utf-8') as file: + data = json.load(file) + + print(f"✅ Loaded {len(data)} total records") + + # Show available robots + available_robots = show_available_robots(data) + + # Filter based on selection + print(f"\n🔍 Filtering robots...") + if USE_INCLUDE_LIST: + if INCLUDE_ROBOTS: + print(f"📋 Including only: {', '.join(INCLUDE_ROBOTS)}") + else: + print("📋 Including ALL robots (no filter)") + else: + if EXCLUDE_ROBOTS: + print(f"❌ Excluding: {', '.join(EXCLUDE_ROBOTS)}") + else: + print("❌ No exclusions (including all)") + + filtered_data = filter_by_robot_selection(data) + print(f"✅ Found {len(filtered_data)} matching records") + + # Save filtered data + print(f"\n💾 Saving to {OUTPUT_FILE}...") + with open(OUTPUT_FILE, 'w', encoding='utf-8') as file: + json.dump(filtered_data, file, indent=2, ensure_ascii=False) + + print(f"✅ Data saved successfully!") + + # Show final statistics + final_robots = {} + for record in filtered_data: + robot_name = str(record["Robô"]).strip() + final_robots[robot_name] = final_robots.get(robot_name, 0) + 1 + + print(f"\n📈 Final robot breakdown:") + print("-" * 30) + for robot, count in sorted(final_robots.items()): + print(f" {robot}: {count} records") + + # Calculate size reduction + original_size = Path(INPUT_FILE).stat().st_size + filtered_size = Path(OUTPUT_FILE).stat().st_size + reduction = ((original_size - filtered_size) / original_size) * 100 + + print(f"\n📊 File size reduction:") + print(f" Original: {original_size:,} bytes") + print(f" Filtered: {filtered_size:,} bytes") + print(f" Reduction: {reduction:.1f}%") + + except FileNotFoundError: + print(f"❌ Error: File '{INPUT_FILE}' not found.") + print(" Please update INPUT_FILE with your actual filename.") + except Exception as e: + print(f"❌ Error: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/separate_robots.py b/separate_robots.py new file mode 100644 index 0000000..d4b18c4 --- /dev/null +++ b/separate_robots.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Script to separate robots data from a larger JSON file to reduce import size. +This script will filter the data to only include records with robot information. +""" + +import json +import sys +from pathlib import Path +from typing import List, Dict, Any + +def load_json_file(file_path: str) -> List[Dict[str, Any]]: + """Load JSON data from file.""" + try: + with open(file_path, 'r', encoding='utf-8') as file: + return json.load(file) + except FileNotFoundError: + print(f"Error: File '{file_path}' not found.") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in file '{file_path}': {e}") + sys.exit(1) + +def filter_robots_data(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Filter data to only include records with robot information.""" + robots_data = [] + + for record in data: + # Check if the record has a "Robô" field and it's not empty + if "Robô" in record and record["Robô"] and str(record["Robô"]).strip(): + robots_data.append(record) + + return robots_data + +def save_robots_data(data: List[Dict[str, Any]], output_file: str): + """Save filtered robots data to a new JSON file.""" + try: + with open(output_file, 'w', encoding='utf-8') as file: + json.dump(data, file, indent=2, ensure_ascii=False) + print(f"Robots data saved to '{output_file}'") + except Exception as e: + print(f"Error saving file: {e}") + sys.exit(1) + +def get_robot_statistics(data: List[Dict[str, Any]]) -> Dict[str, Any]: + """Get statistics about the robots data.""" + if not data: + return {} + + robots = {} + total_records = len(data) + + for record in data: + robot_name = record.get("Robô", "Unknown") + if robot_name not in robots: + robots[robot_name] = 0 + robots[robot_name] += 1 + + return { + "total_records": total_records, + "unique_robots": len(robots), + "robot_counts": robots + } + +def main(): + """Main function to process the JSON file.""" + if len(sys.argv) < 2: + print("Usage: python separate_robots.py [output_file]") + print("Example: python separate_robots.py trades.json robots_only.json") + sys.exit(1) + + input_file = sys.argv[1] + output_file = sys.argv[2] if len(sys.argv) > 2 else "robots_only.json" + + print(f"Processing file: {input_file}") + + # Load the JSON data + print("Loading JSON data...") + data = load_json_file(input_file) + print(f"Loaded {len(data)} total records") + + # Filter for robots data + print("Filtering for robots data...") + robots_data = filter_robots_data(data) + print(f"Found {len(robots_data)} records with robot information") + + # Get statistics + stats = get_robot_statistics(robots_data) + if stats: + print(f"\nRobots Statistics:") + print(f" Total robot records: {stats['total_records']}") + print(f" Unique robots: {stats['unique_robots']}") + print(f" Robot breakdown:") + for robot, count in stats['robot_counts'].items(): + print(f" {robot}: {count} records") + + # Save the filtered data + print(f"\nSaving robots data to: {output_file}") + save_robots_data(robots_data, output_file) + + # Calculate size reduction + original_size = Path(input_file).stat().st_size + filtered_size = Path(output_file).stat().st_size + reduction = ((original_size - filtered_size) / original_size) * 100 + + print(f"\nFile size reduction:") + print(f" Original: {original_size:,} bytes") + print(f" Filtered: {filtered_size:,} bytes") + print(f" Reduction: {reduction:.1f}%") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/simple_robot_filter.py b/simple_robot_filter.py new file mode 100644 index 0000000..a315a01 --- /dev/null +++ b/simple_robot_filter.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +""" +Simple script to filter robots data from JSON file. +Just change the filename below and run! +""" + +import json + +# CHANGE THIS TO YOUR JSON FILENAME +INPUT_FILE = "your_file.json" # Replace with your actual filename +OUTPUT_FILE = "robots_only.json" + +def filter_robots(): + """Filter and save only robots data.""" + try: + # Load the JSON file + print(f"Loading {INPUT_FILE}...") + with open(INPUT_FILE, 'r', encoding='utf-8') as file: + data = json.load(file) + + print(f"Loaded {len(data)} total records") + + # Filter for robots data + robots_data = [] + for record in data: + if "Robô" in record and record["Robô"] and str(record["Robô"]).strip(): + robots_data.append(record) + + print(f"Found {len(robots_data)} records with robot information") + + # Save filtered data + with open(OUTPUT_FILE, 'w', encoding='utf-8') as file: + json.dump(robots_data, file, indent=2, ensure_ascii=False) + + print(f"Robots data saved to {OUTPUT_FILE}") + + # Show robot breakdown + robot_counts = {} + for record in robots_data: + robot = record["Robô"] + robot_counts[robot] = robot_counts.get(robot, 0) + 1 + + print("\nRobot breakdown:") + for robot, count in robot_counts.items(): + print(f" {robot}: {count} records") + + except FileNotFoundError: + print(f"Error: File '{INPUT_FILE}' not found. Please change INPUT_FILE to your actual filename.") + except Exception as e: + print(f"Error: {e}") + +if __name__ == "__main__": + filter_robots() \ No newline at end of file