-
Notifications
You must be signed in to change notification settings - Fork 0
/
webcam_object_detector.ipynb의 사본
1 lines (1 loc) · 18.3 KB
/
webcam_object_detector.ipynb의 사본
1
{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"webcam_object_detector.ipynb의 사본","provenance":[{"file_id":"https://github.com/dhrim/cau_2022_summer/blob/master/material/deep_learning/webcam_object_detector.ipynb","timestamp":1659057259991}],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","gpuClass":"standard"},"cells":[{"cell_type":"code","source":["# import dependencies\n","from IPython.display import display, Javascript, Image\n","from google.colab.output import eval_js\n","from base64 import b64decode, b64encode\n","import cv2\n","import numpy as np\n","import PIL\n","import io\n","import html\n","import time"],"metadata":{"id":"3PpFo2TkPtcS","executionInfo":{"status":"ok","timestamp":1659056941516,"user_tz":-540,"elapsed":625,"user":{"displayName":"","userId":""}}},"execution_count":2,"outputs":[]},{"cell_type":"code","source":["# function to convert the JavaScript object into an OpenCV image\n","def js_to_image(js_reply):\n"," \"\"\"\n"," Params:\n"," js_reply: JavaScript object containing image from webcam\n"," Returns:\n"," img: OpenCV BGR image\n"," \"\"\"\n"," # decode base64 image\n"," image_bytes = b64decode(js_reply.split(',')[1])\n"," # convert bytes to numpy array\n"," jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)\n"," # decode numpy array into OpenCV BGR image\n"," img = cv2.imdecode(jpg_as_np, flags=1)\n","\n"," return img\n","\n","# function to convert OpenCV Rectangle bounding box image into base64 byte string to be overlayed on video stream\n","def bbox_to_bytes(bbox_array):\n"," \"\"\"\n"," Params:\n"," bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.\n"," Returns:\n"," bytes: Base64 image byte string\n"," \"\"\"\n"," # convert array into PIL image\n"," bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGBA')\n"," iobuf = io.BytesIO()\n"," # format bbox into png for return\n"," bbox_PIL.save(iobuf, format='png')\n"," # format return string\n"," bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))\n","\n"," return bbox_bytes"],"metadata":{"id":"DiKiXLJwQLWC","executionInfo":{"status":"ok","timestamp":1659056941517,"user_tz":-540,"elapsed":8,"user":{"displayName":"","userId":""}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["# JavaScript to properly create our live video stream using our webcam as input\n","def video_stream():\n"," js = Javascript('''\n"," var video;\n"," var div = null;\n"," var stream;\n"," var captureCanvas;\n"," var imgElement;\n"," var labelElement;\n"," \n"," var pendingResolve = null;\n"," var shutdown = false;\n"," \n"," function removeDom() {\n"," stream.getVideoTracks()[0].stop();\n"," video.remove();\n"," div.remove();\n"," video = null;\n"," div = null;\n"," stream = null;\n"," imgElement = null;\n"," captureCanvas = null;\n"," labelElement = null;\n"," }\n"," \n"," function onAnimationFrame() {\n"," if (!shutdown) {\n"," window.requestAnimationFrame(onAnimationFrame);\n"," }\n"," if (pendingResolve) {\n"," var result = \"\";\n"," if (!shutdown) {\n"," captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n"," result = captureCanvas.toDataURL('image/jpeg', 0.8)\n"," }\n"," var lp = pendingResolve;\n"," pendingResolve = null;\n"," lp(result);\n"," }\n"," }\n"," \n"," async function createDom() {\n"," if (div !== null) {\n"," return stream;\n"," }\n","\n"," div = document.createElement('div');\n"," div.style.border = '2px solid black';\n"," div.style.padding = '3px';\n"," div.style.width = '100%';\n"," div.style.maxWidth = '600px';\n"," document.body.appendChild(div);\n"," \n"," const modelOut = document.createElement('div');\n"," modelOut.innerHTML = \"<span>Status:</span>\";\n"," labelElement = document.createElement('span');\n"," labelElement.innerText = 'No data';\n"," labelElement.style.fontWeight = 'bold';\n"," modelOut.appendChild(labelElement);\n"," div.appendChild(modelOut);\n"," \n"," video = document.createElement('video');\n"," video.style.display = 'block';\n"," video.width = div.clientWidth - 6;\n"," video.setAttribute('playsinline', '');\n"," video.onclick = () => { shutdown = true; };\n"," stream = await navigator.mediaDevices.getUserMedia(\n"," {video: { facingMode: \"environment\"}});\n"," div.appendChild(video);\n","\n"," imgElement = document.createElement('img');\n"," imgElement.style.position = 'absolute';\n"," imgElement.style.zIndex = 1;\n"," imgElement.onclick = () => { shutdown = true; };\n"," div.appendChild(imgElement);\n"," \n"," const instruction = document.createElement('div');\n"," instruction.innerHTML = \n"," '<span style=\"color: red; font-weight: bold;\">' +\n"," 'When finished, click here or on the video to stop this demo</span>';\n"," div.appendChild(instruction);\n"," instruction.onclick = () => { shutdown = true; };\n"," \n"," video.srcObject = stream;\n"," await video.play();\n","\n"," captureCanvas = document.createElement('canvas');\n"," captureCanvas.width = video.videoWidth;\n"," captureCanvas.height = video.videoHeight;\n"," window.requestAnimationFrame(onAnimationFrame);\n"," \n"," return stream;\n"," }\n"," async function stream_frame(label, imgData) {\n"," if (shutdown) {\n"," removeDom();\n"," shutdown = false;\n"," return '';\n"," }\n","\n"," var preCreate = Date.now();\n"," stream = await createDom();\n"," \n"," var preShow = Date.now();\n"," if (label != \"\") {\n"," labelElement.innerHTML = label;\n"," }\n"," \n"," if (imgData != \"\") {\n"," var videoRect = video.getClientRects()[0];\n"," imgElement.style.top = videoRect.top + \"px\";\n"," imgElement.style.left = videoRect.left + \"px\";\n"," imgElement.style.width = videoRect.width + \"px\";\n"," imgElement.style.height = videoRect.height + \"px\";\n"," imgElement.src = imgData;\n"," }\n"," \n"," var preCapture = Date.now();\n"," var result = await new Promise(function(resolve, reject) {\n"," pendingResolve = resolve;\n"," });\n"," shutdown = false;\n"," \n"," return {'create': preShow - preCreate, \n"," 'show': preCapture - preShow, \n"," 'capture': Date.now() - preCapture,\n"," 'img': result};\n"," }\n"," ''')\n","\n"," display(js)\n"," \n","def video_frame(label, overay_img):\n"," data = eval_js('stream_frame(\"{}\", \"{}\")'.format(label, overay_img))\n"," return data"],"metadata":{"id":"UBg46_OJPw56","executionInfo":{"status":"ok","timestamp":1659056941518,"user_tz":-540,"elapsed":8,"user":{"displayName":"","userId":""}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":[""],"metadata":{"id":"RdHpoUNgc8vg","executionInfo":{"status":"ok","timestamp":1659056941518,"user_tz":-540,"elapsed":7,"user":{"displayName":"","userId":""}}},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":["# 프로젝트 다운로드"],"metadata":{"id":"j2MxAXTLuYlS"}},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wlzvNKuNj9Tw","outputId":"83a07c28-30a9-49c8-9e90-681d9e6c151a","executionInfo":{"status":"ok","timestamp":1659056943004,"user_tz":-540,"elapsed":1492,"user":{"displayName":"","userId":""}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Cloning into 'darkeras-yolov4'...\n","remote: Enumerating objects: 175, done.\u001b[K\n","remote: Counting objects: 100% (36/36), done.\u001b[K\n","remote: Compressing objects: 100% (13/13), done.\u001b[K\n","remote: Total 175 (delta 27), reused 25 (delta 23), pack-reused 139\u001b[K\n","Receiving objects: 100% (175/175), 8.08 MiB | 31.93 MiB/s, done.\n","Resolving deltas: 100% (82/82), done.\n"]}],"source":["!git clone https://github.com/dhrim/darkeras-yolov4"]},{"cell_type":"code","source":["%cd darkeras-yolov4"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2phB4qz-kBbE","outputId":"0ddd66d4-19c4-43fd-c008-a2d85aa1d4c9","executionInfo":{"status":"ok","timestamp":1659056943006,"user_tz":-540,"elapsed":9,"user":{"displayName":"","userId":""}}},"execution_count":6,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/darkeras-yolov4\n"]}]},{"cell_type":"markdown","source":["# 모델 파일 다운로드"],"metadata":{"id":"zvbhKT5aIG8x"}},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/gdrive')"],"metadata":{"id":"09WoTDlsA5Pd","executionInfo":{"status":"ok","timestamp":1659056964219,"user_tz":-540,"elapsed":21218,"user":{"displayName":"","userId":""}},"outputId":"73b1a4c6-6cfa-4d3b-a105-640cc4acb810","colab":{"base_uri":"https://localhost:8080/"}},"execution_count":7,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /gdrive\n"]}]},{"cell_type":"code","source":["!cp /gdrive/MyDrive/tmp/yolov3_custom_last.weights ./"],"metadata":{"id":"ygxYNwikA6Sq","executionInfo":{"status":"ok","timestamp":1659056966897,"user_tz":-540,"elapsed":2692,"user":{"displayName":"","userId":""}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["# !wget -O yolov3.weights https://pjreddie.com/media/files/yolov3.weights"],"metadata":{"id":"0N26Aq6jcqEq","executionInfo":{"status":"ok","timestamp":1659056966898,"user_tz":-540,"elapsed":15,"user":{"displayName":"","userId":""}}},"execution_count":9,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"5f81e3fa"},"source":["# 모델 로딩"]},{"cell_type":"code","source":["import yolov3_wrapper \n","model = yolov3_wrapper.YoloV3Wrapper(\"yolov3_custom_last.weights\", class_num=1)"],"metadata":{"id":"hDORAqZLcsk4","executionInfo":{"status":"ok","timestamp":1659056975258,"user_tz":-540,"elapsed":8373,"user":{"displayName":"","userId":""}}},"execution_count":10,"outputs":[]},{"cell_type":"code","source":["!ls -al"],"metadata":{"id":"5AjKAN-fT233","executionInfo":{"status":"ok","timestamp":1659056975917,"user_tz":-540,"elapsed":685,"user":{"displayName":"","userId":""}},"outputId":"5ef45075-bba5-4546-dd68-701c381580ed","colab":{"base_uri":"https://localhost:8080/"}},"execution_count":11,"outputs":[{"output_type":"stream","name":"stdout","text":["total 244008\n","drwxr-xr-x 9 root root 4096 Jul 29 01:09 .\n","drwxr-xr-x 1 root root 4096 Jul 29 01:09 ..\n","drwxr-xr-x 3 root root 4096 Jul 29 01:09 core\n","drwxr-xr-x 3 root root 4096 Jul 29 01:09 core_yolov4\n","-rw-r--r-- 1 root root 1740766 Jul 29 01:09 darkeras-yolov3.ipynb\n","-rw-r--r-- 1 root root 1745718 Jul 29 01:09 darkeras-yolov4.ipynb\n","drwxr-xr-x 5 root root 4096 Jul 29 01:09 data\n","drwxr-xr-x 2 root root 4096 Jul 29 01:09 docs\n","drwxr-xr-x 8 root root 4096 Jul 29 01:09 .git\n","-rw-r--r-- 1 root root 287 Jul 29 01:09 .gitpod.yml\n","-rw-r--r-- 1 root root 1064 Jul 29 01:09 LICENSE\n","drwxr-xr-x 2 root root 4096 Jul 29 01:09 __pycache__\n","-rw-r--r-- 1 root root 1977 Jul 29 01:09 README.md\n","-rw-r--r-- 1 root root 1751 Jul 29 01:09 requirements.txt\n","drwxr-xr-x 2 root root 4096 Jul 29 01:09 weights\n","-rw------- 1 root root 246305388 Jul 29 01:09 yolov3_custom_last.weights\n","-rw-r--r-- 1 root root 6678 Jul 29 01:09 yolov3_wrapper.py\n"]}]},{"cell_type":"code","source":[""],"metadata":{"id":"ZeRnDWeycHij","executionInfo":{"status":"ok","timestamp":1659056975920,"user_tz":-540,"elapsed":19,"user":{"displayName":"","userId":""}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["import cv2\n","import numpy as np\n","import matplotlib.pyplot as plt"],"metadata":{"id":"0LJ8t70tcG84","executionInfo":{"status":"ok","timestamp":1659056975921,"user_tz":-540,"elapsed":19,"user":{"displayName":"","userId":""}}},"execution_count":12,"outputs":[]},{"cell_type":"code","source":["class Detector:\n","\n"," def detect(self, image):\n"," return model.predict(image)\n","\n","detector = Detector()"],"metadata":{"id":"kKcpBHiUQgut","executionInfo":{"status":"ok","timestamp":1659056975922,"user_tz":-540,"elapsed":19,"user":{"displayName":"","userId":""}}},"execution_count":13,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":512},"id":"yNVYtGEXPq2H","outputId":"910ba08e-85ff-4a4e-e4a1-190a0dc15a0d"},"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.Javascript object>"],"application/javascript":["\n"," var video;\n"," var div = null;\n"," var stream;\n"," var captureCanvas;\n"," var imgElement;\n"," var labelElement;\n"," \n"," var pendingResolve = null;\n"," var shutdown = false;\n"," \n"," function removeDom() {\n"," stream.getVideoTracks()[0].stop();\n"," video.remove();\n"," div.remove();\n"," video = null;\n"," div = null;\n"," stream = null;\n"," imgElement = null;\n"," captureCanvas = null;\n"," labelElement = null;\n"," }\n"," \n"," function onAnimationFrame() {\n"," if (!shutdown) {\n"," window.requestAnimationFrame(onAnimationFrame);\n"," }\n"," if (pendingResolve) {\n"," var result = \"\";\n"," if (!shutdown) {\n"," captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n"," result = captureCanvas.toDataURL('image/jpeg', 0.8)\n"," }\n"," var lp = pendingResolve;\n"," pendingResolve = null;\n"," lp(result);\n"," }\n"," }\n"," \n"," async function createDom() {\n"," if (div !== null) {\n"," return stream;\n"," }\n","\n"," div = document.createElement('div');\n"," div.style.border = '2px solid black';\n"," div.style.padding = '3px';\n"," div.style.width = '100%';\n"," div.style.maxWidth = '600px';\n"," document.body.appendChild(div);\n"," \n"," const modelOut = document.createElement('div');\n"," modelOut.innerHTML = \"<span>Status:</span>\";\n"," labelElement = document.createElement('span');\n"," labelElement.innerText = 'No data';\n"," labelElement.style.fontWeight = 'bold';\n"," modelOut.appendChild(labelElement);\n"," div.appendChild(modelOut);\n"," \n"," video = document.createElement('video');\n"," video.style.display = 'block';\n"," video.width = div.clientWidth - 6;\n"," video.setAttribute('playsinline', '');\n"," video.onclick = () => { shutdown = true; };\n"," stream = await navigator.mediaDevices.getUserMedia(\n"," {video: { facingMode: \"environment\"}});\n"," div.appendChild(video);\n","\n"," imgElement = document.createElement('img');\n"," imgElement.style.position = 'absolute';\n"," imgElement.style.zIndex = 1;\n"," imgElement.onclick = () => { shutdown = true; };\n"," div.appendChild(imgElement);\n"," \n"," const instruction = document.createElement('div');\n"," instruction.innerHTML = \n"," '<span style=\"color: red; font-weight: bold;\">' +\n"," 'When finished, click here or on the video to stop this demo</span>';\n"," div.appendChild(instruction);\n"," instruction.onclick = () => { shutdown = true; };\n"," \n"," video.srcObject = stream;\n"," await video.play();\n","\n"," captureCanvas = document.createElement('canvas');\n"," captureCanvas.width = video.videoWidth;\n"," captureCanvas.height = video.videoHeight;\n"," window.requestAnimationFrame(onAnimationFrame);\n"," \n"," return stream;\n"," }\n"," async function stream_frame(label, imgData) {\n"," if (shutdown) {\n"," removeDom();\n"," shutdown = false;\n"," return '';\n"," }\n","\n"," var preCreate = Date.now();\n"," stream = await createDom();\n"," \n"," var preShow = Date.now();\n"," if (label != \"\") {\n"," labelElement.innerHTML = label;\n"," }\n"," \n"," if (imgData != \"\") {\n"," var videoRect = video.getClientRects()[0];\n"," imgElement.style.top = videoRect.top + \"px\";\n"," imgElement.style.left = videoRect.left + \"px\";\n"," imgElement.style.width = videoRect.width + \"px\";\n"," imgElement.style.height = videoRect.height + \"px\";\n"," imgElement.src = imgData;\n"," }\n"," \n"," var preCapture = Date.now();\n"," var result = await new Promise(function(resolve, reject) {\n"," pendingResolve = resolve;\n"," });\n"," shutdown = false;\n"," \n"," return {'create': preShow - preCreate, \n"," 'show': preCapture - preShow, \n"," 'capture': Date.now() - preCapture,\n"," 'img': result};\n"," }\n"," "]},"metadata":{}}],"source":["# start streaming video from webcam\n","video_stream()\n","\n","# label for video\n","label_html = 'Capturing...'\n","\n","# initialze bounding box to empty\n","overray_bytes = ''\n","count = 0 \n","\n","while True:\n"," js_reply = video_frame(label_html, overray_bytes)\n"," if not js_reply:\n"," break\n"," # convert JS response to OpenCV Image\n"," img = js_to_image(js_reply[\"img\"])\n","\n"," detected_bboxes = detector.detect(img)\n","\n"," # create transparent overlay for bounding box\n"," overray_img = np.zeros([480,640,4], dtype=np.uint8)\n"," # get face bounding box for overlay\n"," for (x1,y1,x2,y2,confidence, category) in detected_bboxes:\n"," x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)\n"," COLOR = (255,0,0)\n"," THICK = 2\n"," overray_img = cv2.rectangle(overray_img,(x1,y1),(x2,y2), COLOR, THICK)\n"," cv2.putText(overray_img, str(confidence), (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, \n"," 1, (0, 0, 255), 1, lineType=cv2.LINE_AA)\n"," \n"," overray_img[:,:,3] = (overray_img.max(axis = 2) > 0 ).astype(int) * 255\n"," # convert overlay of bbox into bytes\n"," overray_bytes = bbox_to_bytes(overray_img)\n"]},{"cell_type":"code","source":[""],"metadata":{"id":"h_2BtA44eGmC"},"execution_count":null,"outputs":[]}]}