特征匹配不再用灰度图，提高稳定性

README.md

@@ -1,16 +1,15 @@
 # JGM Automator
 
-* 基于 https://github.com/Jiahonzheng/JGM-Automator 改进而来
-* 改的地方比较多，我怕合并回去和作者原思路冲突太大，就先放着吧
-* 取消了训练模式的设定，并且将模板匹配改为特征值匹配
-* 提高了搜索速度
-* 不再要求原分辨率图片，理论上对手机或模拟器分辨率只要求16:9
+* 基于 https://github.com/Jiahonzheng/JGM-Automator 改进
+* 基于opencv的特征值匹配 [CSDN博客](https://blog.csdn.net/github_39611196/article/details/81164752)
+* 更改搜索逻辑，大幅提高了搜索速度 
+* 货物不再要求原分辨率图片，理论上对手机或模拟器分辨率只要求16:9
 
 ## 安装与运行
 
 ```bash
 # 安装依赖
-python -m pip install uiautomator2 opencv
+python -m pip install uiautomator2 opencv-python opencv-contrib-python==3.4.2.16
 
 # adb 连接
 # 使用 MuMu 模拟器，确保屏幕大小为 1920（长） * 1080（宽）
@@ -34,9 +33,6 @@ python main.py
 
 <img src="./assets/Screenshot.png" style="zoom:40%" />
 
-+ Weditor
-
-我们可以使用 Weditor 工具，获取屏幕坐标，以及在线编写自动化脚本。
 
 ```bash
 # 安装依赖
@@ -48,4 +44,18 @@ python -m weditor
 
 + 货物素材
 
-我们可以自行制作货物的素材：先生成屏幕快照，~~随后在**实际大小**下~~，截取货物图片，保存至 `targets/` 目录下，并在 `target.py` 声明对应的货物种类及其图片路径。
+我们可以自行制作货物的素材：先生成屏幕快照，~~随后在**实际大小**下~~，截取货物图片，保存至 `targets/` 目录下，并在 `target.py` 声明对应的货物种类及其图片路径。
+
++ 升级列表和收货列表
+  在 `main.py`里，定义这两个列表，即可指定要升级的建筑和要收货的建筑
+  ```py
+   # 升级建筑列表
+    up_list = [(2,1),(3,5)] # 2号升级1次， 3号升级5次
+    # 收货过滤列表
+    harvest_filter = [5,6,7,8] # 只收取5,6,7,8号建筑的货物
+  ```
+
+## 实现细节
++ 截图后，分割右下角，并打上遮罩，提高特征值匹配速度，减少错误：
++ <img src="./targets/test/Figure_1.png" style="zoom:40%" />
++ <img src="./targets/test/Figure_2.png" style="zoom:40%" />

automator.py

@@ -6,58 +6,101 @@
 
 
 class Automator:
-    def __init__(self, device: str, targets: dict):
+    def __init__(self, device: str, targets: dict, upgrade_list: list, harvest_filter:list):
         """
         device: 如果是 USB 连接，则为 adb devices 的返回结果；如果是模拟器，则为模拟器的控制 URL 。
         """
         self.d = u2.connect(device)
         self.targets = targets
+        self.upgrade_list = upgrade_list
+        self.harvest_filter = harvest_filter
+
 
     def start(self):
         """
         启动脚本，请确保已进入游戏页面。
         """
-        trainStop = False # 有可能火车还在进站就截图了，所以等一个周期再收获
+
+        trainStop = False # 有可能火车还在进站就截图了，所以等停稳再收获
+        findSomething = True
+        trainCount = 0
+        '''
+        这段逻辑有点吃屎，我也懒得改了
+        主要实现的是：
+            判断火车是否停稳(上次检测火车不在，这次检测在，那么就是停稳)
+            火车停稳后只搜索固定次数（比如5次），超过搜索次数后就不搜索了，继续收金币和升级
+        '''
         while True:
             # 判断火车
             screen = self.d.screenshot(format="opencv")
+
             if UIMatcher.trainParking(screen):
-                print("Train come!")
-                if trainStop is True:
-                    self._harvest()
+
+                if trainStop is False:
+
+                    time.sleep(0.5) # 等火车停稳
+                    print("[%s] Train come!"%time.asctime())
+                if trainCount < 5 and findSomething:
+                    self._harvest(self.harvest_filter)
+                else:
+                    findSomething = False
+                    self._upgrade(self.upgrade_list)
+                    # 滑动屏幕，收割金币。
+                    self._swipe()
+                    pass
                 trainStop = True
+                trainCount += 1
+                continue
             else:
-                print("Train leave.")
+                print("[%s] No Train."%time.asctime())
+                findSomething = True
                 trainStop = False
+                trainCount = 0
 
 
             # 简单粗暴的方式，处理 “XX之光” 的荣誉显示。
             # 当然，也可以使用图像探测的模式。
             self.d.click(550, 1650)
-            # self._upgrade(2)
-            # self._upgrade(4)
-            self._upgrade(random.randint(1,9))
-            self._upgrade(random.randint(1,9))
-            self._upgrade(random.randint(1,9))
-            self._upgrade(random.randint(1,9))
-
+            self._upgrade(self.upgrade_list)
             # 滑动屏幕，收割金币。
             self._swipe()
 
-    def _upgrade(self,id):
-        self.d.click(0.9, 0.57)
+    def _upgrade_one_with_count(self,id,count):
         sx, sy=self._get_position(id)
         self.d.click(sx, sy)
-        time.sleep(0.5)
-        self.d.click(860/1080, 1760/1920)
-        time.sleep(0.5)
-        self.d.click(1000/1080, 1100/1920)
-
+        time.sleep(0.3)
+        for i in range(count):
+            self.d.click(0.798, 0.884)
+            # time.sleep(0.1)
+
+    def _switch_upgrade_interface(self):
+        self.d.click(0.9, 0.57)
+
+    def _open_upgrade_interface(self):
+        screen = self.d.screenshot(format="opencv")
+        # 判断升级按钮的颜色，蓝比红多就处于正常界面，反之在升级界面
+        R, G, B = UIMatcher.getPixel(screen,0.974,0.615)
+        if B > R:
+            self.d.click(0.9, 0.57)
+
+    def _close_upgrade_interface(self):
+        screen = self.d.screenshot(format="opencv")
+        # 判断升级按钮的颜色，蓝比红多就处于正常界面，反之在升级界面
+        R, G, B = UIMatcher.getPixel(screen,0.974,0.615)
+        if B < R:
+            self.d.click(0.9, 0.57)
 
+    def _upgrade(self, upgrade_list):
+        self._open_upgrade_interface()
+        for building,count in upgrade_list:
+           self._upgrade_one_with_count(building,count) 
+        self._close_upgrade_interface()
+
     def _swipe(self):
         """
         滑动屏幕，收割金币。
         """
+        print("[%s] Swiped."%time.asctime())
         for i in range(3):
             # 横向滑动，共 3 次。
             sx, sy = self._get_position(i * 3 + 1)
@@ -91,31 +134,36 @@ def _get_target_position(self, target: TargetType):
 
 
 
-    def _harvest(self):
+    def _harvest(self, building_filter):
         """
-        探测货物，并搬运货物
-        return: 货物坐标，货物类型
+        探测货物，并搬运货物，过滤想要的建筑位置号
+        @param: 需要收取的坐标列表
+        @return: 货物坐标，货物类型
         """
         # 获取当前屏幕快照
         detected = None
         screen = self.d.screenshot(format="opencv")
 
         for target in TargetType:
-            print("Detecting %s"%target, end="")
+            # print("Detecting %s"%target, end="")
             detected = UIMatcher.match2(screen, target)
             if detected is not None:
-                print("%s is detected."%target)
+                # 如果在过滤列表里
+                if self.targets.get(target) not in building_filter:
+                    print("Skip ---%s---."%target)
+                    continue
+                print("Detected +++%s+++."%target)
                 # 搬运5次
                 for itr in range(5):
-                    self._move_good(self,target, detected)
+                    self._move_good(target, detected)
                 detected = None
-            else:
-                print("")
-
-    @staticmethod       
+            # else:
+            #     # print("")
+
     def _move_good(self, good: TargetType, source):
-        ex, ey = self._get_target_position(good)
-        sx, sy = source
-        self.d.swipe(sx, sy, ex, ey)
-
-
+        try:
+            ex, ey = self._get_target_position(good)
+            sx, sy = source
+            self.d.swipe(sx, sy, ex, ey)
+        except(Exception):
+            pass

cv.py

@@ -1,55 +1,24 @@
 from target import TargetType
-import cv2,time,numpy as np
+import cv2,time,numpy as np,matplotlib.pyplot as plt
 
 
 class UIMatcher:
-    @staticmethod
-    def match(screen, target: TargetType):
-        """
-        在指定快照中确定货物的屏幕位置。
-        """
-        # 获取对应货物的图片。
-        # 有个要点：通过截屏制作货物图片时，请在快照为实际大小的模式下截屏。
-        template = cv2.imread(target.value)
-        x, y = template.shape[0:2]
-        template = cv2.resize(template, (int(y / 2), int(x / 2)))
-        # cv2.imshow(screen,1)
-        height=len(screen)
-        width=len(screen[0])
-        screen = screen[int(0.6*height):height,int(0.5*width):width]
-        # cv2.namedWindow("Image") 
-        # cv2.imshow("Image", screen) 
-        # # cv2.waitKey(0) ti
-        # time.sleep(1)
-        # cv2.destroyAllWindows()
-        # 获取货物图片的宽高。
-        th, tw = template.shape[:2]
-
-        # 调用 OpenCV 模板匹配。
-        res = cv2.matchTemplate(screen, template, cv2.TM_SQDIFF_NORMED)
-        min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
-
-        # 矩形左上角的位置。
-        tl = min_loc
-
-        # 阈值判断。
-        if min_val > 0.15:
-            return None
-
-        # 这里，我随机加入了数字（15），用于补偿匹配值和真实位置的差异。
-        # return (tl[0] + tw / 2 + 15 + 1080*0.5)/1080, (tl[1] + th / 2 + 15 + 1920*0.6)/1920
-        return (tl[0] + tw  + 15 + 1080*0.5)/1080+0.05, (tl[1] + th + 15 + 1920*0.6)/1920+0.1 # 960*540
-
     @staticmethod
     def match2(screen, target: TargetType):
         # 参考 https://blog.csdn.net/github_39611196/article/details/81164752
+
         min_match_count = 5
-        img0 = screen # query image
-        img1 = cv2.imread(target.value,0)  # train image
+        img0 = screen
+        img1 = cv2.imread(target.value)  # train image
         height=len(img0)
         width=len(img0[0])
-        img2 = img0[int(0.65*height):int(0.9*height),int(0.5*width):width]
-
+        img2 = img0[int(0.65*height):int(0.9*height),int(0.5*width):width] # 截取截屏的右下角
+        hh=len(img2)
+        ww=len(img2[0])
+        cover1 = np.array([[[0,0], [ww,0], [ww,int(0.28*hh)], [0,int(hh*0.83)]]], dtype = np.int32) #绘制遮罩1
+        cover2 = np.array([[[ww,hh], [int(ww*0.9),hh], [int(ww*0.2),hh], [ww,int(0.5*hh)]]], dtype = np.int32) #绘制遮罩2
+        img2 = cv2.fillPoly(img2, cover1,  (58,190,149))
+        img2 = cv2.fillPoly(img2, cover2,  (58,190,149))
         # Initiate SIFT detector
         sift = cv2.xfeatures2d.SIFT_create()
         # find the keypoints and descriptors with SIFT
@@ -86,7 +55,7 @@ def match2(screen, target: TargetType):
                 M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
                 matchesMask = mask.ravel().tolist()
                 # 获取原图像的高和宽
-                h, w = img1.shape
+                h, w = len(img1),len(img1[0])
                 # 使用得到的变换矩阵对原图想的四个变换获得在目标图像上的坐标
                 pts = np.float32([[0, 0], [0, h -1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
                 dst = cv2.perspectiveTransform(pts, M)
@@ -96,18 +65,16 @@ def match2(screen, target: TargetType):
                 tary = int(point[0][1] +0.65*height)
                 return tarx/width, tary/height
             except(Exception ):
-                return None
-            # print(tarx,tary)
-            # # 将原图像转换为灰度图
-            # # img2 = cv2.polylines(img2, [np.int32(dst)], True, 255, 3, cv2.LINE_AA)
-            # img4 = cv2.circle(img0,(tarx,tary),10,255,thickness=3)
+                return 0.5,0.5
         else:
-            # print('Not enough matches are found - %d/%d' % (len(good), min_match_count))
-            # matchesMask = None
             return None
 
     @staticmethod
     def trainParking(screen):
+        '''
+        检测火车是否到达，其实是检测铁轨是否还存在
+        @return: 到达:True 没到: False
+        '''
         min_match_count = 5
         img0 = screen # query image
         img1 = cv2.imread('./targets/test/Rail3.jpg',0)  # train image
@@ -124,8 +91,6 @@ def trainParking(screen):
         search_params = dict(checks=50)
         flann = cv2.FlannBasedMatcher(index_params, search_params)
         matches = flann.knnMatch(des1, des2, k=2)
-
-        # store all the good matches as per Lowe's ratio test
         good = []
         for m, n in matches:
             if m.distance < 0.7 * n.distance:
@@ -134,13 +99,13 @@ def trainParking(screen):
         if len(good) > min_match_count:
             return False
         else:
-            # print('Not enough matches are found - %d/%d' % (len(good), min_match_count))
-            # matchesMask = None
             return True
 
     @staticmethod
-    def read(filepath: str):
+    def getPixel(img0, rx, ry):
         """
-        工具函数，用于读取图片。
+        获取某一坐标的RGB值(灰度图会报错)
         """
-        return cv2.imread(filepath)
+        img=cv2.cvtColor(img0, cv2.COLOR_BGR2RGB)
+        return img[int(ry*len(img)), int(rx*len(img[0]))]
+

main.py

@@ -7,19 +7,20 @@
         TargetType.Box: 2,
         TargetType.Dogfood: 2,
         TargetType.Sofa: 3,
-        TargetType.Plant: 3,
-        TargetType.Microphone: 4,
+        TargetType.Plant: 1,
+        TargetType.Microphone: 6,
+        TargetType.Microphone2: 6,
         TargetType.Shoes: 5,
         TargetType.Chicken: 5,
         TargetType.Bottle: 4,
         TargetType.Vegetable: 5,
         TargetType.Food: 8,
-        TargetType.Book: 5,
+        TargetType.Book: 6,
         TargetType.Bag: 6,
         TargetType.Wood: 7,
         TargetType.Oil: 8,
-        TargetType.Food: 8,
-        TargetType.Iron: 9,
+        TargetType.Iron: 7,
+        TargetType.Iron2: 7,
         TargetType.Grass:9,
         TargetType.Tool: 8,
         TargetType.Quilt: 9,
@@ -28,8 +29,11 @@
         TargetType.Cloth: 6
 
     }
-
+    # 升级建筑列表
+    up_list = [(2,1),(3,5)] # 2号升级1次， 3号升级5次
+    # 收货过滤列表
+    harvest_filter = [5,6,7,8] # 只收取5,6,7,8号建筑的货物
     # 连接 adb 。
-    instance = Automator('127.0.0.1:7555', targets)
+    instance = Automator('127.0.0.1:7555', targets, up_list, harvest_filter)
     # 启动脚本。
     instance.start()