|
| 1 | +using Aquality.Selenium.Browsers; |
| 2 | +using Aquality.Selenium.Core.Configurations; |
| 3 | +using OpenCvSharp; |
| 4 | +using OpenQA.Selenium; |
| 5 | +using System; |
| 6 | +using System.Collections.Generic; |
| 7 | +using System.Collections.ObjectModel; |
| 8 | +using System.IO; |
| 9 | +using System.Linq; |
| 10 | + |
| 11 | +namespace Aquality.Selenium.Elements.Interfaces |
| 12 | +{ |
| 13 | + /// <summary> |
| 14 | + /// Locator to search elements by image. |
| 15 | + /// Takes screenshot and finds match using openCV. |
| 16 | + /// Then finds elements by coordinates using JavaScript. |
| 17 | + /// </summary> |
| 18 | + public class ByImage : By, IDisposable |
| 19 | + { |
| 20 | + private readonly Mat template; |
| 21 | + private readonly string description; |
| 22 | + |
| 23 | + /// <summary> |
| 24 | + /// Constructor accepting image file. |
| 25 | + /// </summary> |
| 26 | + /// <param name="file">Image file to locate element by.</param> |
| 27 | + public ByImage(FileInfo file) |
| 28 | + { |
| 29 | + description= file.Name; |
| 30 | + template = new Mat(file.FullName, ImreadModes.Unchanged); |
| 31 | + } |
| 32 | + |
| 33 | + /// <summary> |
| 34 | + /// Constructor accepting image bytes. |
| 35 | + /// </summary> |
| 36 | + /// <param name="bytes">Image bytes to locate element by.</param> |
| 37 | + public ByImage(byte[] bytes) |
| 38 | + { |
| 39 | + description = $"bytes[%d]"; |
| 40 | + template = Mat.ImDecode(bytes, ImreadModes.Unchanged); |
| 41 | + } |
| 42 | + |
| 43 | + /// <summary> |
| 44 | + /// Threshold of image similarity. |
| 45 | + /// Should be a float between 0 and 1, where 1 means 100% match, and 0.5 means 50% match. |
| 46 | + /// </summary> |
| 47 | + public virtual float Threshold { get; set; } = 1 - AqualityServices.Get<IVisualizationConfiguration>().DefaultThreshold; |
| 48 | + |
| 49 | + public override IWebElement FindElement(ISearchContext context) |
| 50 | + { |
| 51 | + return FindElements(context)?.FirstOrDefault() |
| 52 | + ?? throw new NoSuchElementException($"Cannot locate an element using {ToString()}"); |
| 53 | + } |
| 54 | + |
| 55 | + public override ReadOnlyCollection<IWebElement> FindElements(ISearchContext context) |
| 56 | + { |
| 57 | + var source = GetScreenshot(context); |
| 58 | + var result = new Mat(); |
| 59 | + |
| 60 | + Cv2.MatchTemplate(source, template, result, TemplateMatchModes.CCoeffNormed); |
| 61 | + |
| 62 | + Cv2.MinMaxLoc(result, out _, out var maxVal, out _, out var matchLocation); |
| 63 | + var matchCounter = Math.Abs((result.Width - template.Width + 1) * (result.Height - template.Height + 1)); |
| 64 | + var matchLocations = new List<Point>(); |
| 65 | + while (matchCounter > 0 && maxVal >= Threshold) |
| 66 | + { |
| 67 | + matchCounter--; |
| 68 | + matchLocations.Add(matchLocation); |
| 69 | + Cv2.Rectangle(result, new Rect(matchLocation.X, matchLocation.Y, template.Width, template.Height), Scalar.Black, -1); |
| 70 | + Cv2.MinMaxLoc(result, out _, out maxVal, out _, out matchLocation); |
| 71 | + } |
| 72 | + |
| 73 | + return matchLocations.Select(match => GetElementOnPoint(match, context)).ToList().AsReadOnly(); |
| 74 | + } |
| 75 | + |
| 76 | + /// <summary> |
| 77 | + /// Gets a single element on point (find by center coordinates, then select closest to matchLocation). |
| 78 | + /// </summary> |
| 79 | + /// <param name="matchLocation">Location of the upper-left point of the element.</param> |
| 80 | + /// <param name="context">Search context. |
| 81 | + /// If the searchContext is Locatable (like WebElement), will adjust coordinates to be absolute coordinates.</param> |
| 82 | + /// <returns>The closest found element.</returns> |
| 83 | + protected virtual IWebElement GetElementOnPoint(Point matchLocation, ISearchContext context) |
| 84 | + { |
| 85 | + if (context is ILocatable locatable) |
| 86 | + { |
| 87 | + var point = locatable.Coordinates.LocationInDom; |
| 88 | + matchLocation = matchLocation.Add(new Point(point.X, point.Y)); |
| 89 | + } |
| 90 | + var centerLocation = matchLocation.Add(new Point(template.Width / 2, template.Height / 2)); |
| 91 | + |
| 92 | + var elements = AqualityServices.Browser.ExecuteScript<IList<IWebElement>>(JavaScript.GetElementsFromPoint, centerLocation.X, centerLocation.Y) |
| 93 | + .OrderBy(element => DistanceToPoint(matchLocation, element)); |
| 94 | + return elements.First(); |
| 95 | + } |
| 96 | + |
| 97 | + /// <summary> |
| 98 | + /// Calculates distance from element to matching point. |
| 99 | + /// </summary> |
| 100 | + /// <param name="matchLocation">Matching point.</param> |
| 101 | + /// <param name="element">Target element.</param> |
| 102 | + /// <returns>Distance in pixels.</returns> |
| 103 | + protected virtual double DistanceToPoint(Point matchLocation, IWebElement element) |
| 104 | + { |
| 105 | + var elementLocation = element.Location; |
| 106 | + return Math.Sqrt(Math.Pow(matchLocation.X - elementLocation.X, 2) + Math.Pow(matchLocation.Y - elementLocation.Y, 2)); |
| 107 | + } |
| 108 | + |
| 109 | + /// <summary> |
| 110 | + /// Takes screenshot from searchContext if supported, or from browser. |
| 111 | + /// Performs screenshot scaling if devicePixelRatio != 1. |
| 112 | + /// </summary> |
| 113 | + /// <param name="context">Search context for element location.</param> |
| 114 | + /// <returns>Captured screenshot as Mat object.</returns> |
| 115 | + protected virtual Mat GetScreenshot(ISearchContext context) |
| 116 | + { |
| 117 | + var screenshotBytes = context is ITakesScreenshot |
| 118 | + ? (context as ITakesScreenshot).GetScreenshot().AsByteArray |
| 119 | + : AqualityServices.Browser.GetScreenshot(); |
| 120 | + var isBrowserScreenshot = context is IWebDriver || !(context is ITakesScreenshot); |
| 121 | + var source = Mat.ImDecode(screenshotBytes, ImreadModes.Unchanged); |
| 122 | + var devicePixelRatio = AqualityServices.Browser.ExecuteScript<long>(JavaScript.GetDevicePixelRatio); |
| 123 | + if (devicePixelRatio != 1 && isBrowserScreenshot) |
| 124 | + { |
| 125 | + var scaledWidth = (int)(source.Width / devicePixelRatio); |
| 126 | + var scaledHeight = (int)(source.Height / devicePixelRatio); |
| 127 | + Cv2.Resize(source, source, new Size(scaledWidth, scaledHeight), interpolation: InterpolationFlags.Area); |
| 128 | + } |
| 129 | + |
| 130 | + return source; |
| 131 | + } |
| 132 | + |
| 133 | + public override string ToString() |
| 134 | + { |
| 135 | + return $"ByImage: {description}, size: {template.Size()}"; |
| 136 | + } |
| 137 | + |
| 138 | + public override bool Equals(object obj) |
| 139 | + { |
| 140 | + ByImage by = obj as ByImage; |
| 141 | + return by != null && template.ToString().Equals(by.template?.ToString()); |
| 142 | + } |
| 143 | + |
| 144 | + public override int GetHashCode() |
| 145 | + { |
| 146 | + return template.GetHashCode(); |
| 147 | + } |
| 148 | + |
| 149 | + public void Dispose() |
| 150 | + { |
| 151 | + Dispose(true); |
| 152 | + GC.SuppressFinalize(this); |
| 153 | + } |
| 154 | + |
| 155 | + protected virtual void Dispose(bool disposing) |
| 156 | + { |
| 157 | + if (disposing) |
| 158 | + { |
| 159 | + template.Dispose(); |
| 160 | + } |
| 161 | + } |
| 162 | + } |
| 163 | +} |
0 commit comments