Merge pull request avinashkranjan#1989 from SyedImtiyaz-1/ChinesePlate

Chinese Plate Scanner Added

Author: avinashkranjan

Chinese_plate_scan/License-plate-recognition/CNN.py

@@ -0,0 +1,73 @@
+# -*- coding:utf-8 -*-
+# author: DuanshengLiu
+from tensorflow.keras import layers, losses, models
+import numpy as np
+import cv2
+import os
+
+
+def cnn_train():
+    char_dict = {"京": 0, "沪": 1, "津": 2, "渝": 3, "冀": 4, "晋": 5, "蒙": 6, "辽": 7, "吉": 8, "黑": 9, "苏": 10,
+                 "浙": 11, "皖": 12, "闽": 13, "赣": 14, "鲁": 15, "豫": 16, "鄂": 17, "湘": 18, "粤": 19, "桂": 20,
+                 "琼": 21, "川": 22, "贵": 23, "云": 24, "藏": 25, "陕": 26, "甘": 27, "青": 28, "宁": 29, "新": 30,
+                 "0": 31, "1": 32, "2": 33, "3": 34, "4": 35, "5": 36, "6": 37, "7": 38, "8": 39, "9": 40,
+                 "A": 41, "B": 42, "C": 43, "D": 44, "E": 45, "F": 46, "G": 47, "H": 48, "J": 49, "K": 50,
+                 "L": 51, "M": 52, "N": 53, "P": 54, "Q": 55, "R": 56, "S": 57, "T": 58, "U": 59, "V": 60,
+                 "W": 61, "X": 62, "Y": 63, "Z": 64}
+
+    # 读取数据集
+    path = 'home/cnn_datasets/'  # 车牌号数据集路径(车牌图片宽240，高80)
+    pic_name = sorted(os.listdir(path))
+    n = len(pic_name)
+    X_train, y_train = [], []
+    for i in range(n):
+        print("正在读取第%d张图片" % i)
+        img = cv2.imdecode(np.fromfile(path + pic_name[i], dtype=np.uint8), -1)  # cv2.imshow无法读取中文路径图片，改用此方式
+        label = [char_dict[name] for name in pic_name[i][0:7]]  # 图片名前7位为车牌标签
+        X_train.append(img)
+        y_train.append(label)
+    X_train = np.array(X_train)
+    y_train = [np.array(y_train)[:, i] for i in range(7)]  # y_train是长度为7的列表，其中每个都是shape为(n,)的ndarray，分别对应n张图片的第一个字符，第二个字符....第七个字符
+
+    # cnn模型
+    Input = layers.Input((80, 240, 3))  # 车牌图片shape(80,240,3)
+    x = Input
+    x = layers.Conv2D(filters=16, kernel_size=(3, 3), strides=1, padding='same', activation='relu')(x)
+    x = layers.MaxPool2D(pool_size=(2, 2), padding='same', strides=2)(x)
+    for i in range(3):
+        x = layers.Conv2D(filters=32 * 2 ** i, kernel_size=(3, 3), padding='valid', activation='relu')(x)
+        x = layers.Conv2D(filters=32 * 2 ** i, kernel_size=(3, 3), padding='valid', activation='relu')(x)
+        x = layers.MaxPool2D(pool_size=(2, 2), padding='same', strides=2)(x)
+        x = layers.Dropout(0.5)(x)
+    x = layers.Flatten()(x)
+    x = layers.Dropout(0.3)(x)
+    Output = [layers.Dense(65, activation='softmax', name='c%d' % (i + 1))(x) for i in range(7)]  # 7个输出分别对应车牌7个字符，每个输出都为65个类别类概率
+    model = models.Model(inputs=Input, outputs=Output)
+    model.summary()
+    model.compile(optimizer='adam',
+                  loss='sparse_categorical_crossentropy',  # y_train未进行one-hot编码，所以loss选择sparse_categorical_crossentropy
+                  metrics=['accuracy'])
+
+    # 模型训练
+    print("开始训练cnn")
+    model.fit(X_train, y_train, epochs=35)  # 总loss为7个loss的和
+    model.save('cnn.h5')
+    print('cnn.h5保存成功!!!')
+
+
+def cnn_predict(cnn, Lic_img):
+    characters = ["京", "沪", "津", "渝", "冀", "晋", "蒙", "辽", "吉", "黑", "苏", "浙", "皖", "闽", "赣", "鲁", "豫",
+                  "鄂", "湘", "粤", "桂", "琼", "川", "贵", "云", "藏", "陕", "甘", "青", "宁", "新", "0", "1", "2",
+                  "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F", "G", "H", "J", "K", "L", "M",
+                  "N", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]
+    Lic_pred = []
+    for lic in Lic_img:
+        lic_pred = cnn.predict(lic.reshape(1, 80, 240, 3))  # 预测形状应为(1,80,240,3)
+        lic_pred = np.array(lic_pred).reshape(7, 65)  # 列表转为ndarray，形状为(7,65)
+        if len(lic_pred[lic_pred >= 0.8]) >= 4:  # 统计其中预测概率值大于80%以上的个数，大于等于4个以上认为识别率高，识别成功
+            chars = ''
+            for arg in np.argmax(lic_pred, axis=1):  # 取每行中概率值最大的arg,将其转为字符
+                chars += characters[arg]
+            chars = chars[0:2] + '·' + chars[2:]
+            Lic_pred.append((lic, chars))  # 将车牌和识别结果一并存入Lic_pred
+    return Lic_pred

Chinese_plate_scan/License-plate-recognition/README.md

@@ -0,0 +1,15 @@
+# End-to-end-for-chinese-plate-recognition
+
+Chinese license plate recognition software based on U-Net, OpenCV, and CNN, where U-Net and OpenCV are used for license plate localization and correction, and CNN is used for license plate recognition. Both U-Net and CNN are implemented using TensorFlow's Keras.
+
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/lic.png) 
+### Other than that, there are no major issues, and normal recognition works well.
+### Here are some sample result images:
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/0.png)
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/1.png)
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/2.png)
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/3.png)
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/4.png)
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/5.png)
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/6.png)
+![](https://github.com/duanshengliu/End-to-end-for-chinese-plate-recognition/blob/master/test_pic/7.png)

Chinese_plate_scan/License-plate-recognition/UI.py

@@ -0,0 +1,128 @@
+# -*- coding:utf-8 -*-
+# author: DuanshengLiu
+import cv2
+import numpy as np
+from tkinter import *
+from tkinter.filedialog import askopenfilename
+from PIL import Image, ImageTk
+from tensorflow import keras
+from core import locate_and_correct
+from Unet import unet_predict
+from CNN import cnn_predict
+
+
+class Window:
+    def __init__(self, win, ww, wh):
+        self.win = win
+        self.ww = ww
+        self.wh = wh
+        self.win.geometry("%dx%d+%d+%d" % (ww, wh, 200, 50))  # 界面启动时的初始位置
+        self.win.title("车牌定位，矫正和识别软件---by DuanshengLiu")
+        self.img_src_path = None
+
+        self.label_src = Label(self.win, text='原图:', font=('微软雅黑', 13)).place(x=0, y=0)
+        self.label_lic1 = Label(self.win, text='车牌区域1:', font=('微软雅黑', 13)).place(x=615, y=0)
+        self.label_pred1 = Label(self.win, text='识别结果1:', font=('微软雅黑', 13)).place(x=615, y=85)
+        self.label_lic2 = Label(self.win, text='车牌区域2:', font=('微软雅黑', 13)).place(x=615, y=180)
+        self.label_pred2 = Label(self.win, text='识别结果2:', font=('微软雅黑', 13)).place(x=615, y=265)
+        self.label_lic3 = Label(self.win, text='车牌区域3:', font=('微软雅黑', 13)).place(x=615, y=360)
+        self.label_pred3 = Label(self.win, text='识别结果3:', font=('微软雅黑', 13)).place(x=615, y=445)
+
+        self.can_src = Canvas(self.win, width=512, height=512, bg='white', relief='solid', borderwidth=1)  # 原图画布
+        self.can_src.place(x=50, y=0)
+        self.can_lic1 = Canvas(self.win, width=245, height=85, bg='white', relief='solid', borderwidth=1)  # 车牌区域1画布
+        self.can_lic1.place(x=710, y=0)
+        self.can_pred1 = Canvas(self.win, width=245, height=65, bg='white', relief='solid', borderwidth=1)  # 车牌识别1画布
+        self.can_pred1.place(x=710, y=90)
+        self.can_lic2 = Canvas(self.win, width=245, height=85, bg='white', relief='solid', borderwidth=1)  # 车牌区域2画布
+        self.can_lic2.place(x=710, y=175)
+        self.can_pred2 = Canvas(self.win, width=245, height=65, bg='white', relief='solid', borderwidth=1)  # 车牌识别2画布
+        self.can_pred2.place(x=710, y=265)
+        self.can_lic3 = Canvas(self.win, width=245, height=85, bg='white', relief='solid', borderwidth=1)  # 车牌区域3画布
+        self.can_lic3.place(x=710, y=350)
+        self.can_pred3 = Canvas(self.win, width=245, height=65, bg='white', relief='solid', borderwidth=1)  # 车牌识别3画布
+        self.can_pred3.place(x=710, y=440)
+
+        self.button1 = Button(self.win, text='选择文件', width=10, height=1, command=self.load_show_img)  # 选择文件按钮
+        self.button1.place(x=680, y=wh - 30)
+        self.button2 = Button(self.win, text='识别车牌', width=10, height=1, command=self.display)  # 识别车牌按钮
+        self.button2.place(x=780, y=wh - 30)
+        self.button3 = Button(self.win, text='清空所有', width=10, height=1, command=self.clear)  # 清空所有按钮
+        self.button3.place(x=880, y=wh - 30)
+        self.unet = keras.models.load_model('unet.h5')
+        self.cnn = keras.models.load_model('cnn.h5')
+        print('正在启动中,请稍等...')
+        cnn_predict(self.cnn, [np.zeros((80, 240, 3))])
+        print("已启动,开始识别吧！")
+
+
+    def load_show_img(self):
+        self.clear()
+        sv = StringVar()
+        sv.set(askopenfilename())
+        self.img_src_path = Entry(self.win, state='readonly', text=sv).get()  # 获取到所打开的图片
+        img_open = Image.open(self.img_src_path)
+        if img_open.size[0] * img_open.size[1] > 240 * 80:
+            img_open = img_open.resize((512, 512), Image.ANTIALIAS)
+        self.img_Tk = ImageTk.PhotoImage(img_open)
+        self.can_src.create_image(258, 258, image=self.img_Tk, anchor='center')
+
+    def display(self):
+        if self.img_src_path == None:  # 还没选择图片就进行预测
+            self.can_pred1.create_text(32, 15, text='请选择图片', anchor='nw', font=('黑体', 28))
+        else:
+            img_src = cv2.imdecode(np.fromfile(self.img_src_path, dtype=np.uint8), -1)  # 从中文路径读取时用
+            h, w = img_src.shape[0], img_src.shape[1]
+            if h * w <= 240 * 80 and 2 <= w / h <= 5:  # 满足该条件说明可能整个图片就是一张车牌,无需定位,直接识别即可
+                lic = cv2.resize(img_src, dsize=(240, 80), interpolation=cv2.INTER_AREA)[:, :, :3]  # 直接resize为(240,80)
+                img_src_copy, Lic_img = img_src, [lic]
+            else:  # 否则就需通过unet对img_src原图预测,得到img_mask,实现车牌定位,然后进行识别
+                img_src, img_mask = unet_predict(self.unet, self.img_src_path)
+                img_src_copy, Lic_img = locate_and_correct(img_src, img_mask)  # 利用core.py中的locate_and_correct函数进行车牌定位和矫正
+
+            Lic_pred = cnn_predict(self.cnn, Lic_img)  # 利用cnn进行车牌的识别预测,Lic_pred中存的是元祖(车牌图片,识别结果)
+            if Lic_pred:
+                img = Image.fromarray(img_src_copy[:, :, ::-1])  # img_src_copy[:, :, ::-1]将BGR转为RGB
+                self.img_Tk = ImageTk.PhotoImage(img)
+                self.can_src.delete('all')  # 显示前,先清空画板
+                self.can_src.create_image(258, 258, image=self.img_Tk,
+                                          anchor='center')  # img_src_copy上绘制出了定位的车牌轮廓,将其显示在画板上
+                for i, lic_pred in enumerate(Lic_pred):
+                    if i == 0:
+                        self.lic_Tk1 = ImageTk.PhotoImage(Image.fromarray(lic_pred[0][:, :, ::-1]))
+                        self.can_lic1.create_image(5, 5, image=self.lic_Tk1, anchor='nw')
+                        self.can_pred1.create_text(35, 15, text=lic_pred[1], anchor='nw', font=('黑体', 28))
+                    elif i == 1:
+                        self.lic_Tk2 = ImageTk.PhotoImage(Image.fromarray(lic_pred[0][:, :, ::-1]))
+                        self.can_lic2.create_image(5, 5, image=self.lic_Tk2, anchor='nw')
+                        self.can_pred2.create_text(40, 15, text=lic_pred[1], anchor='nw', font=('黑体', 28))
+                    elif i == 2:
+                        self.lic_Tk3 = ImageTk.PhotoImage(Image.fromarray(lic_pred[0][:, :, ::-1]))
+                        self.can_lic3.create_image(5, 5, image=self.lic_Tk3, anchor='nw')
+                        self.can_pred3.create_text(40, 15, text=lic_pred[1], anchor='nw', font=('黑体', 28))
+
+            else:  # Lic_pred为空说明未能识别
+                self.can_pred1.create_text(47, 15, text='未能识别', anchor='nw', font=('黑体', 27))
+
+    def clear(self):
+        self.can_src.delete('all')
+        self.can_lic1.delete('all')
+        self.can_lic2.delete('all')
+        self.can_lic3.delete('all')
+        self.can_pred1.delete('all')
+        self.can_pred2.delete('all')
+        self.can_pred3.delete('all')
+        self.img_src_path = None
+
+    def closeEvent():  # 关闭前清除session(),防止'NoneType' object is not callable
+        keras.backend.clear_session()
+        sys.exit()
+
+
+if __name__ == '__main__':
+    win = Tk()
+    ww = 1000  # 窗口宽设定1000
+    wh = 600  # 窗口高设定600
+    Window(win, ww, wh)
+    win.protocol("WM_DELETE_WINDOW", Window.closeEvent)
+    win.mainloop()

Chinese_plate_scan/License-plate-recognition/Unet.py

@@ -0,0 +1,110 @@
+# -*- coding:utf-8 -*-
+# author: DuanshengLiu
+import numpy as np
+import os
+import cv2
+from tensorflow.keras import layers, losses, models
+
+
+def unet_train():
+    height = 512
+    width = 512
+    path = 'D:/desktop/unet_datasets/'
+    input_name = os.listdir(path + 'train_image')
+    n = len(input_name)
+    print(n)
+    X_train, y_train = [], []
+    for i in range(n):
+        img = cv2.imread(path + 'train_image/%d.png' % i)
+        label = cv2.imread(path + 'train_label/%d.png' % i)
+        X_train.append(img)
+        y_train.append(label)
+    X_train = np.array(X_train)
+    y_train = np.array(y_train)
+
+
+    def Conv2d_BN(x, nb_filter, kernel_size, strides=(1, 1), padding='same'):
+        x = layers.Conv2D(nb_filter, kernel_size, strides=strides, padding=padding)(x)
+        x = layers.BatchNormalization(axis=3)(x)
+        x = layers.LeakyReLU(alpha=0.1)(x)
+        return x
+
+    def Conv2dT_BN(x, filters, kernel_size, strides=(2, 2), padding='same'):
+        x = layers.Conv2DTranspose(filters, kernel_size, strides=strides, padding=padding)(x)
+        x = layers.BatchNormalization(axis=3)(x)
+        x = layers.LeakyReLU(alpha=0.1)(x)
+        return x
+
+    inpt = layers.Input(shape=(height, width, 3))
+    conv1 = Conv2d_BN(inpt, 8, (3, 3))
+    conv1 = Conv2d_BN(conv1, 8, (3, 3))
+    pool1 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1)
+
+    conv2 = Conv2d_BN(pool1, 16, (3, 3))
+    conv2 = Conv2d_BN(conv2, 16, (3, 3))
+    pool2 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv2)
+
+    conv3 = Conv2d_BN(pool2, 32, (3, 3))
+    conv3 = Conv2d_BN(conv3, 32, (3, 3))
+    pool3 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv3)
+
+    conv4 = Conv2d_BN(pool3, 64, (3, 3))
+    conv4 = Conv2d_BN(conv4, 64, (3, 3))
+    pool4 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv4)
+
+    conv5 = Conv2d_BN(pool4, 128, (3, 3))
+    conv5 = layers.Dropout(0.5)(conv5)
+    conv5 = Conv2d_BN(conv5, 128, (3, 3))
+    conv5 = layers.Dropout(0.5)(conv5)
+
+    convt1 = Conv2dT_BN(conv5, 64, (3, 3))
+    concat1 = layers.concatenate([conv4, convt1], axis=3)
+    concat1 = layers.Dropout(0.5)(concat1)
+    conv6 = Conv2d_BN(concat1, 64, (3, 3))
+    conv6 = Conv2d_BN(conv6, 64, (3, 3))
+
+    convt2 = Conv2dT_BN(conv6, 32, (3, 3))
+    concat2 = layers.concatenate([conv3, convt2], axis=3)
+    concat2 = layers.Dropout(0.5)(concat2)
+    conv7 = Conv2d_BN(concat2, 32, (3, 3))
+    conv7 = Conv2d_BN(conv7, 32, (3, 3))
+
+    convt3 = Conv2dT_BN(conv7, 16, (3, 3))
+    concat3 = layers.concatenate([conv2, convt3], axis=3)
+    concat3 = layers.Dropout(0.5)(concat3)
+    conv8 = Conv2d_BN(concat3, 16, (3, 3))
+    conv8 = Conv2d_BN(conv8, 16, (3, 3))
+
+    convt4 = Conv2dT_BN(conv8, 8, (3, 3))
+    concat4 = layers.concatenate([conv1, convt4], axis=3)
+    concat4 = layers.Dropout(0.5)(concat4)
+    conv9 = Conv2d_BN(concat4, 8, (3, 3))
+    conv9 = Conv2d_BN(conv9, 8, (3, 3))
+    conv9 = layers.Dropout(0.5)(conv9)
+    outpt = layers.Conv2D(filters=3, kernel_size=(1, 1), strides=(1, 1), padding='same', activation='relu')(conv9)
+
+    model = models.Model(inpt, outpt)
+    model.compile(optimizer='adam',
+                  loss='mean_squared_error',
+                  metrics=['accuracy'])
+    model.summary()
+
+    print("开始训练u-net")
+    model.fit(X_train, y_train, epochs=100, batch_size=15)
+    model.save('unet.h5')
+    print('unet.h5保存成功!!!')
+
+
+def unet_predict(unet, img_src_path):
+    img_src = cv2.imdecode(np.fromfile(img_src_path, dtype=np.uint8), -1) 
+    if img_src.shape != (512, 512, 3):
+        img_src = cv2.resize(img_src, dsize=(512, 512), interpolation=cv2.INTER_AREA)[:, :, :3]
+    img_src = img_src.reshape(1, 512, 512, 3)
+    img_mask = unet.predict(img_src) 
+    img_src = img_src.reshape(512, 512, 3) 
+    img_mask = img_mask.reshape(512, 512, 3)
+    img_mask = img_mask / np.max(img_mask) * 255  
+    img_mask[:, :, 2] = img_mask[:, :, 1] = img_mask[:, :, 0]  
+    img_mask = img_mask.astype(np.uint8)
+
+    return img_src, img_mask