Merge pull request #1 from waynezhang/master

Fix encoding issue for CJK files

Author: stefanceriu

SCStringsUtility/Others/NSString+SCAdditions.h

@@ -14,4 +14,7 @@
 
 - (NSString*)stringByTrimmingLeadingWhiteSpaces;
 - (NSString*)stringByTrimmingTralingWhiteSpaces;
+
++ (BOOL)detectFileEncoding:(NSStringEncoding *)encoding path:(NSString *)path error:(NSError **)error;
+
 @end

SCStringsUtility/Others/NSString+SCAdditions.m

@@ -51,4 +51,28 @@ - (NSString*)stringByTrimmingTralingWhiteSpaces
     return [self stringByReplacingCharactersInRange:range withString:@""];
 }
 
++ (BOOL)detectFileEncoding:(NSStringEncoding *)encoding path:(NSString *)path error:(NSError **)error
+{
+    NSString *string = [NSString stringWithContentsOfFile:path usedEncoding:encoding error:error];
+    if (string != nil) {
+        if (*encoding == NSUTF16StringEncoding) {
+            // fix for little endian
+            NSFileHandle *fileHandle = [NSFileHandle fileHandleForReadingAtPath:path];
+            [fileHandle seekToFileOffset:0];
+            NSData *data = [fileHandle readDataOfLength:2];
+            const char *head = [data bytes];
+            if (*head == '\xFF' && *(head + 1) == '\xFE') {
+                *encoding = NSUTF16LittleEndianStringEncoding;
+            }
+            [fileHandle closeFile];
+        }
+
+        return YES;
+    } else {
+        SCLog(@"Possible problem in detecting encoding %@", *error);
+
+        return NO;
+    }
+}
+
 @end

SCStringsUtility/Readers&Writers/SCReader.m

@@ -40,9 +40,10 @@ - (id)initWithPath:(NSString*)path
         fileHandle = [NSFileHandle fileHandleForReadingAtPath:self.filePath];
         if (fileHandle == nil) return nil;
 
-        NSError *error;
-        [NSString stringWithContentsOfFile:self.filePath usedEncoding:&encoding error:&error];
-        if(error) SCLog(@"Possible problem in detecting encoding %@",error);
+        if (![NSString detectFileEncoding:&encoding path:filePath error:nil]) {
+            // fallback
+            encoding = NSUTF8StringEncoding;
+        }
 
         lineDelimiter = @"\n";
         filePath = path;
@@ -58,8 +59,8 @@ - (id)initWithPath:(NSString*)path
 - (NSString *)readLine
 {
     if (currentOffset >= totalFileLength) { currentOffset = 0ULL; currentLine = 0; return nil; }
-    
-    NSData * newLineData = [lineDelimiter dataUsingEncoding:NSUTF8StringEncoding];
+
+    NSData * newLineData = [lineDelimiter dataUsingEncoding:self.encoding];
     [fileHandle seekToFileOffset:currentOffset];
     NSMutableData * currentData = [[NSMutableData alloc] init];
     BOOL shouldReadMore = YES;

SCStringsUtility/Readers&Writers/SCStringsWriter.m

@@ -13,11 +13,13 @@
 @interface SCStringsWriter ()
 @property (nonatomic, strong) NSMutableDictionary *fileHandlers;
 @property (nonatomic, strong) NSArray *headers;
+@property(nonatomic, assign) NSStringEncoding fileEncoding;
 @end
 
 @implementation SCStringsWriter
 @synthesize fileHandlers;
 @synthesize headers;
+@synthesize fileEncoding;
 
 - (id)initWithHeaders:(NSArray *)heads
 {
@@ -42,6 +44,11 @@ - (id)initWithTranslationFiles:(NSDictionary *)files
             {
                 NSFileHandle *fileHandle = [NSFileHandle fileHandleForWritingAtPath:[x objectForKey:@"Path"]];
                 if(fileHandle == nil) {SCLog(@"Unable to open file for writing at path %@", [x objectForKey:@"Path"]); return nil;}
+
+                if (![NSString detectFileEncoding:&fileEncoding path:[x objectForKey:@"Path"] error:nil]) {
+                    // fallback
+                    fileEncoding = NSUTF8StringEncoding;
+                }
 
                 [self.fileHandlers setObject:fileHandle forKey:[x objectForKey:@"Language"]];
                 [heads addObject:[x objectForKey:@"Language"]];
@@ -63,7 +70,7 @@ - (void)writeTranslations:(NSDictionary*)translations failure:(void(^)(NSError *
         {
             NSString *comment = [NSString stringWithFormat:@"%@\n", [[translationDict objectForKey:@"Comment"] stringByReplacingOccurrencesOfString:@"\\n" withString:@"\n"]];
             for(NSString *fileHandleKey in self.fileHandlers)
-                [(NSFileHandle*)[self.fileHandlers objectForKey:fileHandleKey] writeData:[comment dataUsingEncoding:NSUTF8StringEncoding]];
+                [(NSFileHandle*)[self.fileHandlers objectForKey:fileHandleKey] writeData:[comment dataUsingEncoding:self.fileEncoding]];
         }
 
         for(NSString *header in self.headers)
@@ -72,7 +79,7 @@ - (void)writeTranslations:(NSDictionary*)translations failure:(void(^)(NSError *
             if(!translation.length) translation = key;
 
             NSString *line = [NSString stringWithFormat:@"\"%@\" = \"%@\";\n\n", key, translation];
-            [(NSFileHandle*)[self.fileHandlers objectForKey:header] writeData:[line dataUsingEncoding:NSUTF8StringEncoding]];
+            [(NSFileHandle*)[self.fileHandlers objectForKey:header] writeData:[line dataUsingEncoding:self.fileEncoding]];
         }
     }