Use UTF8 to encode and decode strings

alejandro-isaza · alejandro-isaza · commit 8517b19ad7b7 · 2019-11-03T09:24:21.000-08:00
diff --git a/Source/Attribute.swift b/Source/Attribute.swift
@@ -29,7 +29,7 @@ open class Attribute {
 
         let pointer = UnsafeMutablePointer<CChar>.allocate(capacity: count + 1)
         H5Aget_name(id, count + 1, pointer)
-        return String(cString: pointer)
+        return String(utf8String: pointer)!
     }
 
     public var space: Dataspace {
diff --git a/Source/Dataset.swift b/Source/Dataset.swift
@@ -91,6 +91,7 @@ extension GroupType {
         precondition(dataspace.dims.count == chunkDimensions.count)
 
         let plist = H5Pcreate(H5P_CLS_DATASET_CREATE_ID_g)
+        H5Pset_char_encoding(plist, H5T_CSET_UTF8)
         let chunkDimensions64 = chunkDimensions.map({ hsize_t(bitPattern: hssize_t($0)) })
         chunkDimensions64.withUnsafeBufferPointer { (pointer) -> Void in
             H5Pset_chunk(plist, Int32(chunkDimensions.count), pointer.baseAddress)
diff --git a/Source/Datatype.swift b/Source/Datatype.swift
@@ -43,6 +43,7 @@ public class Datatype : Object, Equatable {
         let id: hid_t
         if type == String.self {
             id = H5Tcopy(H5T_C_S1_g)
+            H5Tset_cset(id, H5T_CSET_UTF8)
             H5Tset_size(id, -1)
         } else {
             guard let nativeType = NativeType(type: type) else {
@@ -79,6 +80,7 @@ public class Datatype : Object, Equatable {
 
     public class func createString(size: Int = -1) -> Datatype {
         let id = H5Tcopy(H5T_C_S1_g)
+        H5Tset_cset(id, H5T_CSET_UTF8)
         let type = Datatype(id: id)
         type.size = size
         return type
diff --git a/Source/DoubleDataset.swift b/Source/DoubleDataset.swift
@@ -131,6 +131,7 @@ extension GroupType {
         precondition(dataspace.dims.count == chunkDimensions.count)
 
         let plist = H5Pcreate(H5P_CLS_DATASET_CREATE_ID_g)
+        H5Pset_char_encoding(plist, H5T_CSET_UTF8)
         let chunkDimensions64 = chunkDimensions.map({ hsize_t(bitPattern: hssize_t($0)) })
         chunkDimensions64.withUnsafeBufferPointer { (pointer) -> Void in
             H5Pset_chunk(plist, Int32(chunkDimensions.count), pointer.baseAddress)
diff --git a/Source/FloatDataset.swift b/Source/FloatDataset.swift
@@ -131,6 +131,7 @@ extension GroupType {
         precondition(dataspace.dims.count == chunkDimensions.count)
 
         let plist = H5Pcreate(H5P_CLS_DATASET_CREATE_ID_g)
+        H5Pset_char_encoding(plist, H5T_CSET_UTF8)
         let chunkDimensions64 = chunkDimensions.map({ hsize_t(bitPattern: hssize_t($0)) })
         chunkDimensions64.withUnsafeBufferPointer { (pointer) -> Void in
             H5Pset_chunk(plist, Int32(chunkDimensions.count), pointer.baseAddress)
diff --git a/Source/Group.swift b/Source/Group.swift
@@ -55,7 +55,7 @@ public class Group: Object, GroupType {
             let size = H5Gget_objname_by_idx(id, i, nil, 0)
             var name = [Int8](repeating: 0, count: size + 1)
             H5Gget_objname_by_idx(id, i, &name, size + 1)
-            names.append(String(cString: name))
+            names.append(String(utf8String: name)!)
         }
 
         return names
diff --git a/Source/IntDataset.swift b/Source/IntDataset.swift
@@ -131,6 +131,7 @@ extension GroupType {
         precondition(dataspace.dims.count == chunkDimensions.count)
 
         let plist = H5Pcreate(H5P_CLS_DATASET_CREATE_ID_g)
+        H5Pset_char_encoding(plist, H5T_CSET_UTF8)
         let chunkDimensions64 = chunkDimensions.map({ hsize_t(bitPattern: hssize_t($0)) })
         chunkDimensions64.withUnsafeBufferPointer { (pointer) -> Void in
             H5Pset_chunk(plist, Int32(chunkDimensions.count), pointer.baseAddress)
diff --git a/Source/Object.swift b/Source/Object.swift
@@ -34,7 +34,7 @@ open class Object {
 
         let pointer = UnsafeMutablePointer<CChar>.allocate(capacity: count + 1)
         H5Iget_name(id, pointer, count + 1)
-        return String(cString: pointer)
+        return String(utf8String: pointer)!
     }
 }
 
diff --git a/Source/StringAttribute.swift b/Source/StringAttribute.swift
@@ -35,7 +35,7 @@ open class StringAttribute: Attribute {
 
         for pointer in data {
             if let pointer = pointer {
-                strings.append(String(cString: pointer))
+                strings.append(String(utf8String: pointer)!)
             } else {
                 strings.append("")
             }
@@ -65,9 +65,9 @@ open class StringAttribute: Attribute {
         var index = 0
         for _ in 0..<count {
             data.withUnsafeBufferPointer { pointer in
-                let string = String(cString: pointer.baseAddress! + index)
+                let string = String(utf8String: pointer.baseAddress! + index)!
                 strings.append(string)
-                index += string.lengthOfBytes(using: .ascii)
+                index += string.lengthOfBytes(using: .utf8)
                 while index <= size && pointer[index] == 0 {
                     index += 1
                 }
diff --git a/Source/StringDataset.swift b/Source/StringDataset.swift
@@ -79,7 +79,7 @@ public class StringDataset: Dataset {
 
         for pointer in data {
             if let pointer = pointer {
-                strings.append(String(cString: pointer))
+                strings.append(String(utf8String: pointer)!)
             } else {
                 strings.append("")
             }
@@ -100,7 +100,7 @@ public class StringDataset: Dataset {
         let stringSize = size / count
         
         // Add one extra null byte per string entry to ensure zero
-        // padding when parsing to Swift string using String(cString:) below
+        // padding when parsing to Swift string using String(utf8String:) below
         let paddedStringSize = stringSize + 1
         let type = Datatype.createString(size: paddedStringSize)
 
@@ -116,7 +116,7 @@ public class StringDataset: Dataset {
             strings.reserveCapacity(count)
             for idx in 0..<count {
                 let index = idx * type.size
-                let string = String(cString: pointer.baseAddress! + index)
+                let string = String(utf8String: pointer.baseAddress! + index)!
                 strings.append(string)
             }
             return strings
@@ -192,6 +192,7 @@ extension GroupType {
         precondition(dataspace.dims.count == chunkDimensions.count)
 
         let plist = H5Pcreate(H5P_CLS_DATASET_CREATE_ID_g)
+        H5Pset_char_encoding(plist, H5T_CSET_UTF8)
         let chunkDimensions64 = chunkDimensions.map({ hsize_t(bitPattern: hssize_t($0)) })
         chunkDimensions64.withUnsafeBufferPointer { (pointer) -> Void in
             H5Pset_chunk(plist, Int32(chunkDimensions.count), pointer.baseAddress)
diff --git a/Tests/HDF5KitTests/AttributeTests.swift b/Tests/HDF5KitTests/AttributeTests.swift
@@ -48,25 +48,17 @@ class AttributeTests: XCTestCase {
         }
     }
 
-    func testWriteReadString() {
+    func testWriteReadString() throws {
         let filePath = tempFilePath()
         guard let file = File.create(filePath, mode: .truncate) else {
             fatalError("Failed to create file")
         }
         let group = file.createGroup("group")
-        XCTAssertEqual(group.name, "/group")
+        let attribute = try XCTUnwrap(group.createStringAttribute("attribute"))
 
-        guard let attribute = group.createStringAttribute("attribute") else {
-            XCTFail()
-            return
-        }
+        let writeData = "ABCD😀"
+        try attribute.write(writeData)
 
-        do {
-            let writeData = "ABCD"
-            try attribute.write(writeData)
-            XCTAssertEqual(try attribute.read(), [writeData])
-        } catch {
-            XCTFail()
-        }
+        XCTAssertEqual(try attribute.read(), [writeData])
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,7 @@ open class Attribute {`
`29`	`29`
`30`	`30`	`let pointer = UnsafeMutablePointer<CChar>.allocate(capacity: count + 1)`
`31`	`31`	`H5Aget_name(id, count + 1, pointer)`
`32`		`- return String(cString: pointer)`
	`32`	`+ return String(utf8String: pointer)!`
`33`	`33`	`}`
`34`	`34`
`35`	`35`	`public var space: Dataspace {`
Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,7 @@ public class Group: Object, GroupType {`
`55`	`55`	`let size = H5Gget_objname_by_idx(id, i, nil, 0)`
`56`	`56`	`var name = [Int8](repeating: 0, count: size + 1)`
`57`	`57`	`H5Gget_objname_by_idx(id, i, &name, size + 1)`
`58`		`- names.append(String(cString: name))`
	`58`	`+ names.append(String(utf8String: name)!)`
`59`	`59`	`}`
`60`	`60`
`61`	`61`	`return names`
Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ open class Object {`
`34`	`34`
`35`	`35`	`let pointer = UnsafeMutablePointer<CChar>.allocate(capacity: count + 1)`
`36`	`36`	`H5Iget_name(id, pointer, count + 1)`
`37`		`- return String(cString: pointer)`
	`37`	`+ return String(utf8String: pointer)!`
`38`	`38`	`}`
`39`	`39`	`}`
`40`	`40`