Merge pull request #144 from ethscriptions-protocol/fix_collections

Implement Compression and Extraction for Collection JSON Files

Author: RogerPodacter

.gitignore

@@ -33,3 +33,9 @@
 /config/master.key
 
 .DS_Store
+
+# Ignore uncompressed collection JSON files (keep only the tar.gz archive)
+items_by_ethscription.json
+collections_by_name.json
+# Keep the compressed archive
+# collections_data.tar.gz

Dockerfile

@@ -64,7 +64,7 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 
 # Set up non-root user
 RUN useradd rails --create-home --shell /bin/bash && \
-    chown -R rails:rails log tmp storage db config
+    chown -R rails:rails /rails
 USER rails:rails
 
 # Database initialization moved to runtime in entrypoint script

app/models/erc721_ethscriptions_collection_parser.rb

@@ -1,3 +1,6 @@
+require 'zlib'
+require 'rubygems/package'
+
 # Strict parser for the ERC-721 Ethscriptions collection protocol with canonical JSON validation
 class Erc721EthscriptionsCollectionParser
   # Default return for invalid input
@@ -125,6 +128,7 @@ class ValidationError < StandardError; end
 
   DEFAULT_ITEMS_PATH = ENV['COLLECTIONS_ITEMS_PATH'] || Rails.root.join('items_by_ethscription.json')
   DEFAULT_COLLECTIONS_PATH = ENV['COLLECTIONS_META_PATH'] || Rails.root.join('collections_by_name.json')
+  DEFAULT_ARCHIVE_PATH = ENV['COLLECTIONS_ARCHIVE_PATH'] || Rails.root.join('collections_data.tar.gz')
 
   # New API: validate and encode protocol params
   # Unified interface - accepts all possible parameters, uses what it needs
@@ -264,6 +268,47 @@ class << self
     include Memery
 
     def load_import_data(items_path:, collections_path:)
+      archive_path = DEFAULT_ARCHIVE_PATH
+
+      # Check if we need to extract from the tar.gz archive
+      if File.exist?(archive_path)
+        # Check if JSON files don't exist or archive is newer
+        extract_needed = !File.exist?(items_path) || !File.exist?(collections_path) ||
+                        File.mtime(archive_path) > File.mtime(items_path) ||
+                        File.mtime(archive_path) > File.mtime(collections_path)
+
+        if extract_needed
+          Rails.logger.info "Extracting collections data from #{archive_path}" if defined?(Rails)
+
+          # Extract tar.gz archive
+          Zlib::GzipReader.open(archive_path) do |gz|
+            Gem::Package::TarReader.new(gz) do |tar|
+              tar.each do |entry|
+                if entry.file?
+                  case entry.full_name
+                  when 'items_by_ethscription.json'
+                    File.open(items_path, 'wb') do |f|
+                      f.write(entry.read)
+                    end
+                    Rails.logger.info "Extracted #{entry.full_name} to #{items_path}" if defined?(Rails)
+                  when 'collections_by_name.json'
+                    File.open(collections_path, 'wb') do |f|
+                      f.write(entry.read)
+                    end
+                    Rails.logger.info "Extracted #{entry.full_name} to #{collections_path}" if defined?(Rails)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+
+      # Ensure files exist before reading
+      unless File.exist?(items_path) && File.exist?(collections_path)
+        raise "Collections data files not found. Please ensure #{archive_path} exists or provide JSON files directly."
+      end
+
       items = JSON.parse(File.read(items_path))
       collections = JSON.parse(File.read(collections_path))
 
@@ -275,7 +320,7 @@ def load_import_data(items_path:, collections_path:)
       items_by_id.each do |iid, it|
         cname = it['collection_name']
         next unless cname.is_a?(String) && !cname.empty?
-        num = it['ethscription_number'].to_i
+        num = it.fetch('ethscription_number').to_i
         groups[cname] << [iid, num]
       end
 

collections_data.tar.gz

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Script to compress collection JSON files into a single tar.gz archive
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+
+ITEMS_FILE="items_by_ethscription.json"
+COLLECTIONS_FILE="collections_by_name.json"
+ARCHIVE_FILE="collections_data.tar.gz"
+
+# Check if JSON files exist
+if [ ! -f "$ITEMS_FILE" ]; then
+    echo "Error: $ITEMS_FILE not found!"
+    exit 1
+fi
+
+if [ ! -f "$COLLECTIONS_FILE" ]; then
+    echo "Error: $COLLECTIONS_FILE not found!"
+    exit 1
+fi
+
+# Create tar.gz archive
+echo "Creating $ARCHIVE_FILE..."
+tar -czf "$ARCHIVE_FILE" "$ITEMS_FILE" "$COLLECTIONS_FILE"
+
+if [ $? -eq 0 ]; then
+    echo "Successfully created $ARCHIVE_FILE"
+
+    # Show file sizes for comparison
+    echo ""
+    echo "File sizes:"
+    ls -lh "$ITEMS_FILE" "$COLLECTIONS_FILE" "$ARCHIVE_FILE" | awk '{print $9 ": " $5}'
+
+    # Calculate compression ratio (macOS compatible)
+    ORIGINAL_SIZE=$(stat -f %z "$ITEMS_FILE" "$COLLECTIONS_FILE" 2>/dev/null | awk '{sum += $1} END {print sum}')
+    if [ -z "$ORIGINAL_SIZE" ]; then
+        # Linux fallback
+        ORIGINAL_SIZE=$(stat -c %s "$ITEMS_FILE" "$COLLECTIONS_FILE" 2>/dev/null | awk '{sum += $1} END {print sum}')
+    fi
+    COMPRESSED_SIZE=$(stat -f %z "$ARCHIVE_FILE" 2>/dev/null || stat -c %s "$ARCHIVE_FILE" 2>/dev/null)
+    if [ -n "$ORIGINAL_SIZE" ] && [ -n "$COMPRESSED_SIZE" ]; then
+        RATIO=$(echo "scale=2; (1 - $COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100" | bc)
+    else
+        RATIO="N/A"
+    fi
+
+    echo ""
+    echo "Compression ratio: ${RATIO}% size reduction"
+else
+    echo "Error: Failed to create archive"
+    exit 1
+fi

compress_collections.sh

@@ -168,8 +168,8 @@ contract ERC20FixedDenominationManager is IProtocolHandler {
 
         bytes memory collectionInitCalldata = abi.encodeWithSelector(
             ERC721EthscriptionsCollection.initialize.selector,
-            string.concat(deployOp.tick, " ERC-721"),  // Collection name
-            string.concat(deployOp.tick.upper(), "-ERC-721"),  // Collection symbol
+            deployOp.tick,  // Collection name
+            deployOp.tick.upper(),  // Collection symbol
             address(this),  // Manager owns the collection
             ethscriptionId  // Collection ID is the deploy ethscription ID
         );

contracts/src/ERC20FixedDenominationManager.sol

@@ -197,7 +197,7 @@ contract NameRegistry is ERC721EthscriptionsEnumerableUpgradeable, IProtocolHand
         bytes memory json = abi.encodePacked(
             '{"name":"',
             name.escapeJSON(),
-            '","description":"An Ethscriptions name"',
+            '","description":"An Ethscription name"',
             ',"ethscription_id":"',
             ethscriptionIdHex,
             '","ethscription_number":',
@@ -222,7 +222,7 @@ contract NameRegistry is ERC721EthscriptionsEnumerableUpgradeable, IProtocolHand
         return string(abi.encodePacked(
             'data:application/json;base64,',
             Base64.encode(bytes(
-                '{"name":"Word Domains Registry",'
+                '{"name":"Ethscription Names",'
                 '"description":"On-chain name system for Ethscriptions. Allowed characters: a-z, 0-9, and _ (underscore). Max length: 30 characters.",'
                 '"image":"data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTAwIiBoZWlnaHQ9IjUwMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48cmVjdCB3aWR0aD0iNTAwIiBoZWlnaHQ9IjUwMCIgZmlsbD0iIzEwMTAxMCIvPjx0ZXh0IHg9IjI1MCIgeT0iMjUwIiBmb250LXNpemU9IjgwIiBmb250LWZhbWlseT0ibW9ub3NwYWNlIiB0ZXh0LWFuY2hvcj0ibWlkZGxlIiBmaWxsPSIjMDBmZjAwIj5bTkFNRVNdPC90ZXh0Pjwvc3ZnPg==",'
                 '"external_link":"https://ethscriptions.com"}'

contracts/src/NameRegistry.sol

@@ -639,8 +639,8 @@ contract EthscriptionsTokenTest is TestSetup {
 
         // Verify collection properties
         ERC721EthscriptionsCollection collection = ERC721EthscriptionsCollection(collectionAddr);
-        assertEq(collection.name(), "COLL ERC-721");
-        assertEq(collection.symbol(), "COLL-ERC-721");
+        assertEq(collection.name(), "COLL");
+        assertEq(collection.symbol(), "COLL");
         assertEq(collection.collectionId(), DEPLOY_TX_HASH);
 
         // Verify collection lookups work

contracts/test/EthscriptionsToken.t.sol

@@ -77,6 +77,8 @@ def self.parse_log(log)
       parse_items_removed(log)
     when 'CollectionEdited'
       parse_collection_edited(log)
+    when 'OwnershipTransferred'
+      parse_collection_ownership_transferred(log)
     else
       {
         event: event_name,
@@ -370,6 +372,15 @@ def self.parse_collection_edited(log)
     }
   end
 
+  def self.parse_collection_ownership_transferred(log)
+    {
+      event: 'OwnershipTransferred',
+      collection_id: log['topics'][1],
+      previous_owner: log['topics'][2] ? '0x' + log['topics'][2][-40..] : nil,
+      new_owner: log['topics'][3] ? '0x' + log['topics'][3][-40..] : nil
+    }
+  end
+
   # Helper to check if a receipt contains a successful protocol execution
   def self.protocol_succeeded?(receipt)
     events = parse_receipt_events(receipt)