diff --git a/assets/share/item/relics/RELICS_SAVAGE_AREA.png b/assets/share/item/relics/RELICS_SAVAGE_AREA.png new file mode 100644 index 000000000..c2735d6cc Binary files /dev/null and b/assets/share/item/relics/RELICS_SAVAGE_AREA.png differ diff --git a/module/boundary_detection/utils.py b/module/boundary_detection/utils.py new file mode 100644 index 000000000..ee4073462 --- /dev/null +++ b/module/boundary_detection/utils.py @@ -0,0 +1,171 @@ +import math +import numpy as np +import cv2 +import datetime + +from module.base import utils + +def _crop_image_with_object_area(image, area): + """ + Crop the image with only object area. + It will help filter lines outside the object area + + Args: + image (np.array): + Target Image + area (tuple): + tuple of length 4 representing width_min, height_min, width_max, height_max + + Returns: + the cropped image + """ + height, width, channels = image.shape + cropped_image = image[area[1]:area[3], area[0]:area[2], 0:channels] + return cropped_image + + +def _covert_hough_lines(lines, height_min, width_min): + """ + Covert the hough lines from cv2 from represented by rho, theta to represented by 2 points. + + Args: + lines (np.array): + Shape of the lines is num_lines, 1, 2 + height_min (int): + Min height of the area boundary + width_min (int): + Min width of the area boundary + + Returns: + Converted lines with each line [x1, y1, x2, y2] + """ + if lines is None or len(lines) == 0: + return [] + + num_lines = np.shape(lines)[0] + rho = lines[:, :, 0].reshape(num_lines) + theta = lines[:, :, 1].reshape(num_lines) + a = np.cos(theta) + b = np.sin(theta) + x0 = a * rho + y0 = b * rho + x1 = (x0 + 1000*(-b) + width_min) + y1 = (y0 + 1000*(a) + height_min) + x2 = (x0 - 1000*(-b) + width_min) + y2 = (y0 - 1000*(a) + height_min) + return np.stack([x1, y1, x2, y2], axis=1).astype(int) + + +def _mask_image_hsv(cropped_image): + """ + Mask target colors using hsv on the target regions + + Args: + cropped_image (np.array): + Cropped image with + + Remarks: + Objects are off 5 colors: orange, purple, blue, green, gray + We filter the color based on the hsv of the five colors + + Returns: + Mask image with only target colors highlighted + """ + hsv_image = cv2.cvtColor(cropped_image, cv2.COLOR_RGB2HSV) + + orange_low = np.array([2, 73, 107]) + orange_high = np.array([19, 122, 222]) + orange_mask = cv2.inRange(hsv_image, orange_low, orange_high) + + green_low = np.array([85, 80, 75]) + green_high = np.array([111, 141, 180]) + green_mask = cv2.inRange(hsv_image, green_low, green_high) + + blue_low = np.array([105, 82, 92]) + blue_high = np.array([120, 174, 205]) + blue_mask = cv2.inRange(hsv_image, blue_low, blue_high) + + purple_low = np.array([114, 64, 82]) + purple_high = np.array([136, 142, 220]) + purple_mask = cv2.inRange(hsv_image, purple_low, purple_high) + + gray_low = np.array([109, 0, 55]) + gray_high = np.array([135, 68, 190]) + gray_mask = cv2.inRange(hsv_image, gray_low, gray_high) + + mask = orange_mask + blue_mask + purple_mask + green_mask + gray_mask + print(f"HSV Mask: {np.shape(mask)}") + print(f"Mask: {mask}") + return cv2.bitwise_and(cropped_image, cropped_image, mask=mask) + + +def find_hough_lines(image, area): + """ + Find the boundary lines of the objects with hough algorithm + + Args: + image (np.array): + target image + area (tuple): + tuple of length 4 representing width_min, height_min, width_max, height_max + + Returns: + hough lines divided into horizontal ones and vertical ones + """ + cropped_image = _crop_image_with_object_area(image, area) + masked_image = _mask_image_hsv(cropped_image) + gray = cv2.cvtColor(masked_image, cv2.COLOR_RGB2GRAY) + _, threshold_image = cv2.threshold(gray, 60, 255, cv2.THRESH_BINARY) + edges = cv2.Canny(threshold_image, 20, 150, apertureSize=3) + lines_h = _covert_hough_lines(cv2.HoughLines(edges, 1, np.pi/180, 200), area[1], area[0]) + lines_v = _covert_hough_lines(cv2.HoughLines(edges, 1, np.pi/180, 110), area[1], area[0]) + + lines_result_h = [] + h_axis = [] + for line in lines_h: + if abs(line[1]-line[3]) < 2 and not any(abs(prev_line[1] - line[1]) < 5 for prev_line in lines_result_h): + lines_result_h.append(line) + h_axis.append(line[1]) + + lines_result_v = [] + v_axis = [] + for line in lines_v: + if abs(line[0]-line[2]) < 2 and not any(abs(prev_line[0] - line[0]) < 5 for prev_line in lines_result_v): + lines_result_v.append(line) + v_axis.append(line[0]) + + return h_axis, v_axis + + +def get_object_rectangles(image, area): + """ + Find the boundary rectangles of the objects + + Args: + image (np.array): + target image + area (tuple): + tuple of length 4 representing width_min, height_min, width_max, height_max + + Returns: + hough lines divided into horizontal ones and vertical ones + """ + lines_h, lines_v = find_hough_lines(image, area) + lines_h.sort() + lines_v.sort() + + rec_h_pair = [] + for h_index in range(1, len(lines_h)-1): + if abs((lines_h[h_index] - lines_h[h_index-1]) - 89) < 2 and abs((lines_h[h_index+1] - lines_h[h_index]) - 20) < 2: + rec_h_pair.append((lines_h[h_index-1], lines_h[h_index+1])) + + rec_v_pair = [] + for v_index in range(1, len(lines_v)): + if abs((lines_v[v_index] - lines_v[v_index-1]) - 96) < 2: + rec_v_pair.append((lines_v[v_index-1], lines_v[v_index])) + + for h_pair in rec_h_pair: + for v_pair in rec_v_pair: + cv2.rectangle(image, (v_pair[0], h_pair[0]), (v_pair[1], h_pair[1]), (0, 0, 255), 2) + + return rec_h_pair, rec_v_pair diff --git a/tasks/item/assets/assets_item_relics.py b/tasks/item/assets/assets_item_relics.py index d4060b601..037236e65 100644 --- a/tasks/item/assets/assets_item_relics.py +++ b/tasks/item/assets/assets_item_relics.py @@ -53,6 +53,16 @@ button=(720, 649, 738, 667), ), ) +RELICS_SAVAGE_AREA = ButtonWrapper( + name='RELICS_SAVAGE_AREA', + share=Button( + file='./assets/share/item/relics/RELICS_SAVAGE_AREA.png', + area=(421, 136, 1165, 530), + search=(401, 116, 1185, 550), + color=(79, 72, 98), + button=(421, 136, 1165, 530), + ), +) SALVAGE = ButtonWrapper( name='SALVAGE', share=Button( diff --git a/tasks/item/relics.py b/tasks/item/relics.py index 2c72b905a..e3f729cdb 100644 --- a/tasks/item/relics.py +++ b/tasks/item/relics.py @@ -1,3 +1,4 @@ +from module.boundary_detection import utils from module.base.timer import Timer from module.logger import logger from tasks.base.assets.assets_base_page import CLOSE @@ -6,11 +7,14 @@ from tasks.item.keywords import KEYWORD_ITEM_TAB from tasks.item.ui import ItemUI +import time class RelicsUI(ItemUI): + def _is_in_salvage(self) -> bool: return self.appear(ORDER_ASCENDING) or self.appear(ORDER_DESCENDING) + def salvage_relic(self, skip_first_screenshot=True) -> bool: logger.hr('Salvage Relic', level=2) self.item_goto(KEYWORD_ITEM_TAB.Relics, wait_until_stable=False) @@ -28,23 +32,47 @@ def salvage_relic(self, skip_first_screenshot=True) -> bool: skip_first_screenshot = True interval = Timer(1) + relics_selected_count = 0 + relics_selected_sign = None while 1: # salvage -> first relic selected + logger.info("Start Iteration") if skip_first_screenshot: skip_first_screenshot = False else: self.device.screenshot() + h_bound, v_bound = utils.get_object_rectangles(self.device.image, RELICS_SAVAGE_AREA.area) + + if len(h_bound) == 0 or len(v_bound) == 0: + continue + + relics_v_index = relics_selected_count % len(v_bound) + relics_h_index = (int)((relics_selected_count - relics_v_index) / len(v_bound)) + + relics_v = v_bound[relics_v_index] + relics_h = h_bound[relics_h_index] + relic= RelicsUI._get_relics_button(relics_v, relics_h) + # The first frame entering relic page, SALVAGE is a white button as it's the default state. # At the second frame, SALVAGE is disabled since no items are selected. # So here uses the minus button on the first relic. - if self.image_color_count(FIRST_RELIC_SELECTED, color=(245, 245, 245), threshold=221, count=50): + if relics_selected_sign is not None and self.image_color_count(relics_selected_sign, color=(245, 245, 245), threshold=221, count=100): logger.info('First relic selected') break + if self.appear_then_click(ORDER_DESCENDING, interval=2): continue + if interval.reached() and self.appear(ORDER_ASCENDING) \ - and self.image_color_count(FIRST_RELIC, (233, 192, 108)): - self.device.click(FIRST_RELIC) + and self.image_color_count(relic, (233, 192, 108)): + self.device.click(relic) + + relics_selected_count += 1 + logger.info(f"Trying to find the savagable relics for the {relics_selected_count} time") + + time.sleep(3) + relics_selected_sign = RelicsUI._get_relics_selected_button(relics_v, relics_h) + interval.reset() continue @@ -82,3 +110,29 @@ def salvage_relic(self, skip_first_screenshot=True) -> bool: interval.reset() continue return True + + + @staticmethod + def _get_relics_button(relics_v_bound, relics_h_bound): + area = (relics_v_bound[0], relics_h_bound[0], relics_v_bound[1], relics_h_bound[1]) + search = (area[0] - 20, area[1] - 20, area[2] + 20, area[3] + 20) + return Button( + file='./assets/share/item/relics/FIRST_RELIC.png', + area=area, + search=search, + color=(72, 92, 124), + button=area, + ) + + + @staticmethod + def _get_relics_selected_button(relics_v_bound, relics_h_bound): + area = (relics_v_bound[0] - 10, relics_h_bound[0] - 24, relics_v_bound[0] + 18, relics_h_bound[0] + 4) + search = (area[0] - 20, area[1] - 20, area[2] + 20, area[3] + 20) + return Button( + file='./assets/share/item/relics/FIRST_RELIC_SELECTED.png', + area=area, + search=search, + color=(193, 194, 198), + button=area, + )