Skip to content
This repository was archived by the owner on Dec 7, 2020. It is now read-only.

Commit 8499f53

Browse files
author
Severi Haverila
committed
moved more functionality to library, dont use appium for screenshot
1 parent 88b20f5 commit 8499f53

File tree

4 files changed

+175
-132
lines changed

4 files changed

+175
-132
lines changed

‎src/main/java/AbstractAppiumTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -162,11 +162,6 @@ public static boolean idevicescreenshotCheck() throws IOException, InterruptedEx
162162
return idevicescreenshotExists;
163163
}
164164

165-
//Stops the script for the given amount of seconds.
166-
public static void sleep(int seconds) throws InterruptedException {
167-
Thread.sleep(seconds * 1000);
168-
}
169-
170165
//Stops the script for the given amount of seconds.
171166
public static void sleep(double seconds) throws Exception {
172167
log("Waiting for " + seconds + " sec");

‎src/main/java/TestdroidImageRecognition.java

Lines changed: 20 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public class TestdroidImageRecognition extends AbstractAppiumTest {
2323

2424
AkazeImageFinder imageFinder = new AkazeImageFinder();
2525

26-
private String queryimageFolder = "";
26+
private String queryImageSubFolder = "";
2727
public boolean found = false;
2828

2929

@@ -33,7 +33,7 @@ public void setQueryImageFolder() {
3333
Dimension size = driver.manage().window().getSize();
3434
log("Screen size: " + size.toString());
3535
if ((size.getHeight() <= 500) || (size.getWidth() <= 500)) {
36-
queryimageFolder = "low_res/";
36+
queryImageSubFolder = "low_res/";
3737
}
3838
}
3939

@@ -43,83 +43,36 @@ public void setQueryImageFolder() {
4343
* FINDING AN IMAGE ON SCREEN
4444
* ======================================================================================
4545
*/
46+
4647

47-
48+
public Point[] findImageOnScreen(String image) throws Exception {
49+
ImageRecognitionSettingsDTO defaultSettings = new ImageRecognitionSettingsDTO();
50+
return findImageOnScreen(image, defaultSettings);
51+
}
52+
4853
public Point[] findImageOnScreen(String image, ImageRecognitionSettingsDTO settings) throws Exception {
4954
ImageSearchDTO dto = findImageOnScreen2(image, settings);
5055
return dto.getImageRectangle();
5156
}
5257

53-
public ImageSearchDTO findImageOnScreen2(String image, ImageRecognitionSettingsDTO settings) throws Exception {
54-
ImageSearchDTO foundImage = findImageLoop(image, settings, getScreenSizeADB());
55-
if (foundImage.isFound() && settings.isCrop()) {
56-
cropImage(foundImage);
57-
}
58+
private ImageSearchDTO findImageOnScreen2(String imageName, ImageRecognitionSettingsDTO settings) throws Exception {
59+
// queryImageFolder is "", unless set by setQueryImageFolder()
60+
String queryImageFolder = "queryimages/" + queryImageSubFolder;
61+
String screenshotsFolder = "target/reports/screenshots/"; //TODO Severi remove this
62+
ImageSearchDTO foundImage = ImageRecognition.findImageOnScreen(imageName, queryImageFolder, screenshotsFolder, settings, getScreenSizeADB(), platformName);
5863
return foundImage;
5964
}
6065

66+
public void waitForImageToDisappearFromScreen(String image) throws Exception {
67+
String queryImageFolder = "queryimages/" + queryImageSubFolder; //TODO Severi remove this
68+
String screenshotsFolder = "target/reports/screenshots/"; //TODO Severi remove this
69+
Dimension screenSize = getScreenSizeADB(); //TODO Severi remove this
70+
boolean hasImageDisappeared = ImageRecognition.hasImageDissappearedFromScreenBeforeTimeout(image, queryImageFolder, screenshotsFolder, screenSize, platformName);
71+
assert(hasImageDisappeared);
72+
}
6173

62-
private void cropImage(ImageSearchDTO foundImage) {
63-
Point[] imgRect = foundImage.getImageRectangle();
64-
Point top_left = imgRect[0];
65-
Point top_right = imgRect[1];
66-
Point bottom_left = imgRect[2];
67-
Point center = imgRect[4];
68-
imageFinder.cropImage(foundImage.getScreenshotFile(), top_left.x, top_left.y, top_right.x - top_left.x, bottom_left.y - top_left.y);
69-
}
70-
71-
private ImageSearchDTO findImageLoop(String image, ImageRecognitionSettingsDTO settings, Dimension screenSize) throws InterruptedException, IOException, Exception {
72-
long start_time = System.nanoTime();
73-
ImageSearchDTO foundImageDto = new ImageSearchDTO();
74-
for (int i = 0; i < settings.getRetries(); i++) {
75-
// queryImageFolder is "", unless set by setQueryImageFolder()
76-
String queryImageFile = "queryimages/" + queryimageFolder + image;
77-
String screenshotFile = takeScreenshot(image + "_screenshot");
78-
Point[] imgRect = ImageRecognition.findImage(queryImageFile, screenshotFile, settings, platformName, screenSize);
79-
if (imgRect!=null){
80-
long end_time = System.nanoTime();
81-
int difference = (int) ((end_time - start_time) / 1e6 / 1000);
82-
log("==> Find image took: " + difference + " secs.");
83-
foundImageDto.setImageRectangle(imgRect);
84-
foundImageDto.setScreenshotFile(screenshotFile);
85-
return foundImageDto;
86-
}
87-
retryWait(settings);
88-
}
89-
log("==> Image not found");
90-
return foundImageDto;
91-
}
92-
93-
private void retryWait(ImageRecognitionSettingsDTO settings) throws InterruptedException {
94-
if (settings.getRetryWaitTime() > 0) {
95-
log("retryWait given, sleeping " + settings.getRetryWaitTime() + " seconds.");
96-
sleep(settings.getRetryWaitTime());
97-
}
98-
}
9974

100-
public Point[] findImageOnScreen(String image) throws Exception {
101-
ImageRecognitionSettingsDTO defaultSettings = new ImageRecognitionSettingsDTO();
102-
return findImageOnScreen(image, defaultSettings);
103-
}
10475

105-
public boolean waitForImageToDisappearFromScreen(String image) throws Exception {
106-
long start = System.nanoTime();
107-
log("==> Trying to find image: " + image);
108-
int retry_counter=0;
109-
String queryImageFile = "queryimages/" + queryimageFolder + image;
110-
Dimension screenSize = getScreenSizeADB();
111-
112-
while (((System.nanoTime() - start) / 1e6 / 1000 < 300)) {
113-
String screenShotFile = takeScreenshot(image + "_screenshot_"+retry_counter);
114-
if ((ImageRecognition.findImage(queryImageFile, screenShotFile, platformName, screenSize)) == null) {
115-
log("Image has successfully disappeared from screen.");
116-
return true;
117-
}
118-
sleep(3);
119-
}
120-
logger.warn("Image did not disappear from screen");
121-
return false;
122-
}
12376

12477

12578
/**
@@ -225,65 +178,6 @@ public String grabTextFromImage(String image) throws Exception {
225178
}
226179

227180

228-
public String takeScreenshot(String screenshotName) throws IOException, InterruptedException {
229-
if (idevicescreenshotExists) {
230-
// Keep Appium session alive between multiple non-driver screenshots
231-
driver.manage().window().getSize();
232-
}
233-
234-
long start_time = System.nanoTime();
235-
String screenshotFile = screenshotsFolder + screenshotName + ".png";
236-
String fullFileName = System.getProperty("user.dir") + "/" + screenshotFile;
237-
238-
if (platformName.equalsIgnoreCase("iOS")) {
239-
takeIDeviceScreenshot(fullFileName);
240-
} else {
241-
takeAndroidScreenshot(fullFileName);
242-
}
243-
long end_time = System.nanoTime();
244-
int difference = (int) ((end_time - start_time) / 1e6 / 1000);
245-
logger.info("==> Taking a screenshot took " + difference + " secs.");
246-
return screenshotFile;
247-
}
248-
249-
private void takeAndroidScreenshot(String fullFileName) throws IOException, InterruptedException {
250-
log("Taking android screenshot...");
251-
log(fullFileName);
252-
String[] cmd = new String[]{"screenshot2", "-d", fullFileName};
253-
Process p = Runtime.getRuntime().exec(cmd);
254-
BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));
255-
String line;
256-
while ((line = in.readLine()) != null)
257-
log(line);
258-
259-
int exitVal = p.waitFor();
260-
if (exitVal != 0) {
261-
log("screenshot2 process exited with value: " + exitVal);
262-
}
263-
}
264-
265-
266-
267-
268-
private static void takeIDeviceScreenshot(String fullFileName) throws IOException, InterruptedException {
269-
String[] cmd = new String[]{"idevicescreenshot", "-u", udid, fullFileName};
270-
Process p = Runtime.getRuntime().exec(cmd);
271-
BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));
272-
String line;
273-
while ((line = in.readLine()) != null)
274-
log(line);
275-
276-
int exitVal = p.waitFor();
277-
if (exitVal != 0) {
278-
log("idevicescreenshot process exited with value: " + exitVal);
279-
}
280-
cmd = new String[]{"sips", "-s", "format", "png", fullFileName, "--out", fullFileName};
281-
p = Runtime.getRuntime().exec(cmd);
282-
exitVal = p.waitFor();
283-
if (exitVal != 0) {
284-
log("sips process exited with value: " + exitVal);
285-
}
286-
}
287181

288182

289183

‎src/main/java/library/ImageRecognition.java

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
package library;
22

33
import java.io.BufferedReader;
4+
import java.io.IOException;
45
import java.io.InputStream;
56
import java.io.InputStreamReader;
67

8+
import org.apache.commons.io.FilenameUtils;
79
import org.opencv.core.Point;
810
import org.openqa.selenium.Dimension;
911
import org.slf4j.Logger;
1012
import org.slf4j.LoggerFactory;
1113

1214

1315
import dtos.ImageRecognitionSettingsDTO;
16+
import dtos.ImageSearchDTO;
1417

1518
public class ImageRecognition {
1619

@@ -107,6 +110,32 @@ private static Point[] scaleImageRectangleForIos(Dimension screenSize, Point[] i
107110
private static boolean isPointInsideScreenBounds(Point center, Dimension screenSize) {
108111
return !((center.x >= screenSize.width) || (center.x < 0) || (center.y >= screenSize.height) || (center.y < 0));
109112
}
113+
114+
115+
public static boolean hasImageDissappearedFromScreenBeforeTimeout(String image, String queryImageFolder,
116+
String screenshotsFolder, Dimension screenSize, String platformName) throws Exception {
117+
log("==> Trying to find image: " + image);
118+
int retry_counter=0;
119+
long start = System.nanoTime();
120+
String queryImageFile = queryImageFolder + image;
121+
while (((System.nanoTime() - start) / 1e6 / 1000 < 300)) {
122+
String screenshotName = parseFileName(image) + "_screenshot_"+retry_counter;
123+
String screenShotFile = ImageRecognition.takeScreenshot(screenshotName, screenshotsFolder, platformName);
124+
if ((findImage(queryImageFile, screenShotFile, platformName, screenSize)) == null) {
125+
log("Image has successfully disappeared from screen.");
126+
return true;
127+
}
128+
sleep(3);
129+
retry_counter++;
130+
}
131+
logger.warn("Image did not disappear from screen");
132+
return false;
133+
}
134+
135+
private static String parseFileName(String imageFile){
136+
return FilenameUtils.getBaseName(imageFile);
137+
}
138+
110139

111140

112141
public static String getTextStringFromImage(String imageInput) {
@@ -133,4 +162,129 @@ public static String getTextStringFromImage(String imageInput) {
133162

134163

135164

165+
166+
public static ImageSearchDTO findImageOnScreen(String image, String queryImageFolder, String screenshotsFolder, ImageRecognitionSettingsDTO settings, Dimension screenSize, String platformName) throws InterruptedException, IOException, Exception {
167+
ImageSearchDTO foundImageDto = findImageLoop(image, queryImageFolder, screenshotsFolder, settings, screenSize, platformName);
168+
if (foundImageDto.isFound() && settings.isCrop()) {
169+
cropImage(foundImageDto);
170+
}
171+
return foundImageDto;
172+
}
173+
174+
private static ImageSearchDTO findImageLoop(String image, String queryImageFolder, String screenshotsFolder, ImageRecognitionSettingsDTO settings, Dimension screenSize, String platformName) throws InterruptedException, IOException, Exception {
175+
long start_time = System.nanoTime();
176+
ImageSearchDTO foundImageDto = new ImageSearchDTO();
177+
for (int i = 0; i < settings.getRetries(); i++) {
178+
String queryImageFile = queryImageFolder + image;
179+
String screenshotFile = takeScreenshot(image + "_screenshot",screenshotsFolder, platformName);
180+
Point[] imgRect = ImageRecognition.findImage(queryImageFile, screenshotFile, settings, platformName, screenSize);
181+
if (imgRect!=null){
182+
long end_time = System.nanoTime();
183+
int difference = (int) ((end_time - start_time) / 1e6 / 1000);
184+
log("==> Find image took: " + difference + " secs.");
185+
foundImageDto.setImageRectangle(imgRect);
186+
foundImageDto.setScreenshotFile(screenshotFile);
187+
return foundImageDto;
188+
}
189+
retryWait(settings);
190+
}
191+
log("==> Image not found");
192+
return foundImageDto;
193+
}
194+
195+
private static void cropImage(ImageSearchDTO foundImage) {
196+
Point[] imgRect = foundImage.getImageRectangle();
197+
Point top_left = imgRect[0];
198+
Point top_right = imgRect[1];
199+
Point bottom_left = imgRect[2];
200+
Point center = imgRect[4];
201+
imageFinder.cropImage(foundImage.getScreenshotFile(), top_left.x, top_left.y, top_right.x - top_left.x, bottom_left.y - top_left.y);
202+
}
203+
204+
private static void retryWait(ImageRecognitionSettingsDTO settings) throws InterruptedException {
205+
if (settings.getRetryWaitTime() > 0) {
206+
log("retryWait given, sleeping " + settings.getRetryWaitTime() + " seconds.");
207+
sleep(settings.getRetryWaitTime());
208+
}
209+
}
210+
211+
//Stops the script for the given amount of seconds.
212+
private static void sleep(int seconds) throws InterruptedException {
213+
Thread.sleep(seconds * 1000);
214+
}
215+
216+
217+
218+
219+
/* TODO
220+
* if (idevicescreenshotExists) {
221+
// Keep Appium session alive between multiple non-driver screenshots
222+
driver.manage().window().getSize();
223+
}
224+
225+
*/
226+
public static String takeScreenshot(String screenshotName, String screenshotsFolder, String platformName) throws Exception {
227+
long start_time = System.nanoTime();
228+
229+
String screenshotFile = screenshotsFolder + screenshotName + ".png";
230+
String fullFileName = System.getProperty("user.dir") + "/" + screenshotFile;
231+
232+
if (platformName.equalsIgnoreCase("iOS")) {
233+
takeIDeviceScreenshot(fullFileName);
234+
} else if (platformName.equalsIgnoreCase("Android")) {
235+
takeAndroidScreenshot(fullFileName);
236+
} else{
237+
throw new Exception("Invalid platformName: "+platformName);
238+
}
239+
240+
long end_time = System.nanoTime();
241+
int difference = (int) ((end_time - start_time) / 1e6 / 1000);
242+
logger.info("==> Taking a screenshot took " + difference + " secs.");
243+
return screenshotFile;
244+
}
245+
246+
247+
private static void takeAndroidScreenshot(String fullFileName) throws IOException, InterruptedException {
248+
log("Taking android screenshot...");
249+
log(fullFileName);
250+
String[] cmd = new String[]{"screenshot2", "-d", fullFileName};
251+
Process p = Runtime.getRuntime().exec(cmd);
252+
BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));
253+
String line;
254+
while ((line = in.readLine()) != null)
255+
log(line);
256+
257+
int exitVal = p.waitFor();
258+
if (exitVal != 0) {
259+
log("screenshot2 process exited with value: " + exitVal);
260+
}
261+
}
262+
263+
264+
265+
266+
private static void takeIDeviceScreenshot(String fullFileName) throws IOException, InterruptedException {
267+
String udid = System.getenv("UDID");
268+
String[] cmd = new String[]{"idevicescreenshot", "-u", udid, fullFileName};
269+
Process p = Runtime.getRuntime().exec(cmd);
270+
BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));
271+
String line;
272+
while ((line = in.readLine()) != null)
273+
log(line);
274+
275+
int exitVal = p.waitFor();
276+
if (exitVal != 0) {
277+
log("idevicescreenshot process exited with value: " + exitVal);
278+
}
279+
cmd = new String[]{"sips", "-s", "format", "png", fullFileName, "--out", fullFileName};
280+
p = Runtime.getRuntime().exec(cmd);
281+
exitVal = p.waitFor();
282+
if (exitVal != 0) {
283+
log("sips process exited with value: " + exitVal);
284+
}
285+
}
286+
287+
288+
289+
136290
}

‎src/test/java/AndroidSample.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public static void tearDown() {
5151
public void mainPageTest() throws Exception {
5252
driver.manage().timeouts().implicitlyWait(60, TimeUnit.SECONDS);
5353
log("Image Recognition sample script started.");
54-
takeScreenshot("Before hideKeyboard");
54+
// takeScreenshot("Before hideKeyboard");
5555

5656
AppiumCommons.hideKeyboard(driver);
5757

0 commit comments

Comments
 (0)