Report precent difference when using a tolerance

FBSnapshotTestCase/Categories/UIImage+Compare.h

@@ -32,6 +32,13 @@
 
 @interface UIImage (Compare)
 
+/// Takes a tolerance percentage (0.0-1.0) and compares this image with another image. Returns YES if the images differ less than the tollerance.
 - (BOOL)fb_compareWithImage:(UIImage *)image tolerance:(CGFloat)tolerance;
 
+/// Performs a bitmap comparison of this image to another image. Returns YES if the images are exactly the same.
+- (BOOL)fb_isEqualToImage:(UIImage *)image;
+
+/// Returns the percent of total pixels that differ between this image and another image as a float ranging from 0.0 to 1.0.
+- (CGFloat)fb_differenceFromImage:(UIImage *)image;
+
 @end

FBSnapshotTestCase/Categories/UIImage+Compare.m

@@ -46,89 +46,64 @@ @implementation UIImage (Compare)
 
 - (BOOL)fb_compareWithImage:(UIImage *)image tolerance:(CGFloat)tolerance
 {
+  return [self fb_isEqualToImage:image] || (tolerance > 0.0 && [self fb_differenceFromImage:image] < tolerance);
+}
+
+
+- (BOOL)fb_isEqualToImage:(UIImage *)image {
   NSAssert(CGSizeEqualToSize(self.size, image.size), @"Images must be same size.");
 
-  CGSize referenceImageSize = CGSizeMake(CGImageGetWidth(self.CGImage), CGImageGetHeight(self.CGImage));
-  CGSize imageSize = CGSizeMake(CGImageGetWidth(image.CGImage), CGImageGetHeight(image.CGImage));
-
-  // The images have the equal size, so we could use the smallest amount of bytes because of byte padding
-  size_t minBytesPerRow = MIN(CGImageGetBytesPerRow(self.CGImage), CGImageGetBytesPerRow(image.CGImage));
-  size_t referenceImageSizeBytes = referenceImageSize.height * minBytesPerRow;
-  void *referenceImagePixels = calloc(1, referenceImageSizeBytes);
-  void *imagePixels = calloc(1, referenceImageSizeBytes);
-
-  if (!referenceImagePixels || !imagePixels) {
-    free(referenceImagePixels);
-    free(imagePixels);
-    return NO;
-  }
+  CGContextRef referenceContext = [self fb_bitmapContext];
+  CGContextRef imageContext = [image fb_bitmapContext];
 
-  CGContextRef referenceImageContext = CGBitmapContextCreate(referenceImagePixels,
-                                                             referenceImageSize.width,
-                                                             referenceImageSize.height,
-                                                             CGImageGetBitsPerComponent(self.CGImage),
-                                                             minBytesPerRow,
-                                                             CGImageGetColorSpace(self.CGImage),
-                                                             (CGBitmapInfo)kCGImageAlphaPremultipliedLast
-                                                             );
-  CGContextRef imageContext = CGBitmapContextCreate(imagePixels,
-                                                    imageSize.width,
-                                                    imageSize.height,
-                                                    CGImageGetBitsPerComponent(image.CGImage),
-                                                    minBytesPerRow,
-                                                    CGImageGetColorSpace(image.CGImage),
-                                                    (CGBitmapInfo)kCGImageAlphaPremultipliedLast
-                                                    );
-
-  if (!referenceImageContext || !imageContext) {
-    CGContextRelease(referenceImageContext);
-    CGContextRelease(imageContext);
-    free(referenceImagePixels);
-    free(imagePixels);
-    return NO;
-  }
-
-  CGContextDrawImage(referenceImageContext, CGRectMake(0, 0, referenceImageSize.width, referenceImageSize.height), self.CGImage);
-  CGContextDrawImage(imageContext, CGRectMake(0, 0, imageSize.width, imageSize.height), image.CGImage);
-
-  CGContextRelease(referenceImageContext);
+  size_t pixelCount = CGBitmapContextGetHeight(referenceContext) * CGBitmapContextGetBytesPerRow(referenceContext);
+  BOOL matches = (memcmp(CGBitmapContextGetData(referenceContext), CGBitmapContextGetData(imageContext), pixelCount) == 0);
+
+  CGContextRelease(referenceContext);
   CGContextRelease(imageContext);
+
+  return matches;
+}
 
-  BOOL imageEqual = YES;
-
-  // Do a fast compare if we can
-  if (tolerance == 0) {
-    imageEqual = (memcmp(referenceImagePixels, imagePixels, referenceImageSizeBytes) == 0);
-  } else {
-    // Go through each pixel in turn and see if it is different
-    const NSInteger pixelCount = referenceImageSize.width * referenceImageSize.height;
-
-    FBComparePixel *p1 = referenceImagePixels;
-    FBComparePixel *p2 = imagePixels;
-
-    NSInteger numDiffPixels = 0;
-    for (int n = 0; n < pixelCount; ++n) {
-      // If this pixel is different, increment the pixel diff count and see
-      // if we have hit our limit.
-      if (p1->raw != p2->raw) {
-        numDiffPixels ++;
-
-        CGFloat percent = (CGFloat)numDiffPixels / pixelCount;
-        if (percent > tolerance) {
-          imageEqual = NO;
-          break;
-        }
-      }
+- (CGFloat)fb_differenceFromImage:(UIImage *)image {
+  NSAssert(CGSizeEqualToSize(self.size, image.size), @"Images must be same size.");
 
-      p1++;
-      p2++;
+  // Go through each pixel in turn and see if it is different
+  CGContextRef referenceContext = [self fb_bitmapContext];
+  CGContextRef imageContext = [image fb_bitmapContext];
+
+  FBComparePixel *p1 = CGBitmapContextGetData(referenceContext);
+  FBComparePixel *p2 = CGBitmapContextGetData(imageContext);
+
+  NSInteger pixelCount = CGBitmapContextGetWidth(referenceContext) * CGBitmapContextGetHeight(referenceContext);
+  NSInteger numDiffPixels = 0;
+  for (int n = 0; n < pixelCount; ++n) {
+    // If this pixel is different, increment the pixel diff count and see
+    // if we have hit our limit.
+    if (p1->raw != p2->raw) {
+      numDiffPixels++;
     }
+
+    p1++;
+    p2++;
   }
+
+  return (CGFloat)numDiffPixels / pixelCount;
+}
 
-  free(referenceImagePixels);
-  free(imagePixels);
-
-  return imageEqual;
+- (CGContextRef)fb_bitmapContext {
+  CGContextRef context = CGBitmapContextCreate(NULL,
+                                               self.size.width,
+                                               self.size.height,
+                                               CGImageGetBitsPerComponent(self.CGImage),
+                                               CGImageGetBytesPerRow(self.CGImage),
+                                               CGImageGetColorSpace(self.CGImage),
+                                               (CGBitmapInfo)kCGImageAlphaPremultipliedLast);
+
+  CGContextDrawImage(context, (CGRect){.size = self.size}, self.CGImage);
+
+  NSAssert(context != nil, @"Unable to create context for comparision.");
+  return context;
 }
 
 @end

FBSnapshotTestCase/FBSnapshotTestController.h

@@ -43,6 +43,11 @@ extern NSString *const FBCapturedImageKey;
  */
 extern NSString *const FBDiffedImageKey;
 
+/**
+ Errors returned by the methods of FBSnapshotTestController will contain this key if a tolerance was specified.
+ */
+extern NSString *const FBPercentDifferenceKey;
+
 /**
  Provides the heavy-lifting for FBSnapshotTestCase. It loads and saves images, along with performing the actual pixel-
  by-pixel comparison of images.

FBSnapshotTestCase/FBSnapshotTestController.m

@@ -21,6 +21,7 @@
 NSString *const FBReferenceImageKey = @"FBReferenceImageKey";
 NSString *const FBCapturedImageKey = @"FBCapturedImageKey";
 NSString *const FBDiffedImageKey = @"FBDiffedImageKey";
+NSString *const FBPercentDifferenceKey = @"FBPercentDifferenceKey";
 
 typedef NS_ENUM(NSUInteger, FBTestSnapshotFileNameType) {
   FBTestSnapshotFileNameTypeReference,
@@ -130,25 +131,44 @@ - (BOOL)compareReferenceImage:(UIImage *)referenceImage
                         error:(NSError **)errorPtr
 {
   BOOL sameImageDimensions = CGSizeEqualToSize(referenceImage.size, image.size);
-  if (sameImageDimensions && [referenceImage fb_compareWithImage:image tolerance:tolerance]) {
+  if (sameImageDimensions && [referenceImage fb_isEqualToImage:image]) {
     return YES;
   }
 
+  CGFloat percentDifference = 0.0;
+  if (tolerance > 0.0) {
+    percentDifference = [referenceImage fb_differenceFromImage:image];
+    if (percentDifference < tolerance) {
+      return YES;
+    }
+  }
+
   if (NULL != errorPtr) {
     NSString *errorDescription = sameImageDimensions ? @"Images different" : @"Images different sizes";
-    NSString *errorReason = sameImageDimensions ? [NSString stringWithFormat:@"image pixels differed by more than %.2f%% from the reference image", tolerance * 100]
+    NSString *errorReason = sameImageDimensions ? @"Images differed"
                                                 : [NSString stringWithFormat:@"referenceImage:%@, image:%@", NSStringFromCGSize(referenceImage.size), NSStringFromCGSize(image.size)];
     FBSnapshotTestControllerErrorCode errorCode = sameImageDimensions ? FBSnapshotTestControllerErrorCodeImagesDifferent : FBSnapshotTestControllerErrorCodeImagesDifferentSizes;
 
-    *errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
-                                    code:errorCode
-                                userInfo:@{
-                                           NSLocalizedDescriptionKey: errorDescription,
-                                           NSLocalizedFailureReasonErrorKey: errorReason,
-                                           FBReferenceImageKey: referenceImage,
-                                           FBCapturedImageKey: image,
-                                           FBDiffedImageKey: [referenceImage fb_diffWithImage:image],
-                                           }];
+    if (sameImageDimensions && tolerance > 0.0) {
+      *errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
+                                      code:errorCode
+                                  userInfo:@{
+                                             NSLocalizedDescriptionKey: errorDescription,
+                                             NSLocalizedFailureReasonErrorKey: [NSString stringWithFormat:@"Images differed by %.2f%% from the reference (with a tolerance of %.2f%%)", percentDifference * 100, tolerance * 100],
+                                             FBReferenceImageKey: referenceImage,
+                                             FBCapturedImageKey: image,
+                                             FBPercentDifferenceKey: @(percentDifference * 100),
+                                             }];
+    } else {
+      *errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
+                                      code:errorCode
+                                  userInfo:@{
+                                             NSLocalizedDescriptionKey: errorDescription,
+                                             NSLocalizedFailureReasonErrorKey: errorReason,
+                                             FBReferenceImageKey: referenceImage,
+                                             FBCapturedImageKey: image,
+                                             }];
+    }
   }
   return NO;
 }

FBSnapshotTestCaseTests/FBSnapshotControllerTests.m

@@ -60,6 +60,7 @@ - (void)testCompareReferenceImageWithVeryLowToleranceShouldNotMatch
     XCTAssertFalse([controller compareReferenceImage:referenceImage toImage:testImage tolerance:0.0001 error:&error]);
     XCTAssertNotNil(error);
     XCTAssertEqual(error.code, FBSnapshotTestControllerErrorCodeImagesDifferent);
+    XCTAssertEqual([error.userInfo[FBPercentDifferenceKey] doubleValue], .04);
 }
 
 - (void)testCompareReferenceImageWithVeryLowToleranceShouldMatch