Add video streaming support, changes in UI layout

Author: stanislau-arkhipenka

requirements.txt

@@ -5,4 +5,5 @@ click
 RPi.GPIO
 PySimpleGUI
 thrift
-busio
+busio
+opencv-python

steam_deck/controller.py

@@ -58,6 +58,7 @@ def __init__(self, thrift_client: Client) -> None:
         self.monitor_dev()
         self.log_queue = queue.Queue()
 
+
     def await_controller(self) -> str:
         logger.info("Waiting for controller")
         while True:
@@ -75,8 +76,6 @@ def monitor_dev(self) -> None:
         self.monitor_thread.start()
 
 
-
-
     def _monitor_dev(self, vibrate: bool = False) -> None:   # TODO vibrate not implemented
         while True:
             if self._terminate_monitor:
@@ -112,4 +111,5 @@ def _monitor_dev(self, vibrate: bool = False) -> None:   # TODO vibrate not impl
     def terminate(self) -> None:
         self._terminate_monitor = True
         self.monitor_thread.join()
+        self.thrift_client = None
         return

steam_deck/network.py

@@ -46,4 +46,4 @@ def connect(host: str, port: int):
     client = LockedClient(protocol)
     transport.open()
 
-    return client
+    return socket, client

steam_deck/ui.py

@@ -1,8 +1,10 @@
 import logging
 import time
 import PySimpleGUI as sg
+
 from controller import SteamDeckController
-from network import connect, Client
+from video_stream import VideoStream
+from network import connect, LockedClient
 from typing import Any, Dict, List, Optional
 from spec.ttypes import ARR_status, Mode, Giat
 import queue
@@ -16,8 +18,11 @@ class SteamDeckUI:
     def __init__(self):
         self.connected = False
         self.network_client: Client = None
-        self._connect_layout = self.get_connect_layout()
-        self._op_layout = self.get_op_layout()
+        self.network_socket = None
+        self.video_stream: VideoStream = None
+        self.logs_view = True
+        self._connect_layout = None
+        self._op_layout = None
 
         self.ui_state: Dict[str, Any] = {}
         self.robot_state = ARR_status(
@@ -33,13 +38,12 @@ def __init__(self):
         self.event_routes = {
             "_button_connect": self._button_connect,
             "_button_disconnect": self._button_disconnect,
-            "_button_ping": self._button_ping
+            "_button_ping": self._button_ping,
+            "_button_view": self._button_view
         }
 
-        self._multiline_log = None
         self.controller: Optional[SteamDeckController] = None
 
-
     def init_connect_window(self):
         self._connect_layout = self.get_connect_layout()
         self.window = sg.Window("Connect", self._connect_layout, finalize=True, font=self.FONT)
@@ -52,16 +56,20 @@ def init_op_window(self):
     def get_connect_layout(self) -> List[List[Any]]:
         return [
             [
+                sg.Text("RC: "),
                 sg.Input('localhost', key="_input_ip_addr", size=32),
                 sg.Input('9090', key="_input_port", size=5),
             ],
+            [
+                sg.Text("VD: "),
+                sg.Input('http://192.168.0.6:9999/stream', key='_input_video_addr', size=38)
+            ],
             [
                 sg.Button("Connect", key="_button_connect"),
             ],
         ]
 
     def get_op_layout(self) -> List[List[Any]]:
-        self._multiline_log = sg.Multiline('', size=(64,18), key="_multiline_log", autoscroll=True)
         return [
             [
                 sg.Text('M: N/A', key="_text_mode"),
@@ -72,14 +80,13 @@ def get_op_layout(self) -> List[List[Any]]:
                 sg.Text('Battery: N/A', key="_text_battery"),
             ],
             [
-                # TODO insert image here
-                self._multiline_log
+                sg.Image(key="_image_stream", visible =(not self.logs_view)),
+                sg.Multiline('', size=(64,18), key="_multiline_log", autoscroll=True, visible=self.logs_view)
             ],
             [
                 sg.Text("Connection quality: TBD"),
                 sg.Button("Ping", key="_button_ping"),
-                sg.Button("Video: OFF"),
-                sg.Button("Logs: OFF"),
+                sg.Button("View: LOGS", key="_button_view"),
                 sg.Button("Disconnect", key="_button_disconnect")
             ]
         ]
@@ -96,33 +103,63 @@ def _button_connect(self):
         if not self.connected:
             host = self.ui_state["_input_ip_addr"]
             port = int(self.ui_state["_input_port"])
-            self.network_client = connect(host, port)
+            video_addr =self.ui_state["_input_video_addr"]
+            self.network_socket, self.network_client = connect(host, port)
             self.connected = True
             logger.info("Connected to %s:%s", host, port)
             self.window.close()
             self.init_op_window()
             self.controller = SteamDeckController(self.network_client)
+            self.video_stream = VideoStream(video_addr)
+            if not self.logs_view:
+                self.video_stream.enable()
         else:
             logger.warning("Already connected!")
 
+
     def _button_disconnect(self):
         if self.connected:
-            # TODO do disconnect logic here! Close Socket/etc
+            self.controller.terminate()
+            self.controller = None
+            self.network_socket.close()
+            self.network_client = None
+            self.video_stream.disable()
             self.window.close()
             self.init_connect_window()
             self.connected = False
             logger.info("Disconnected")
         else:
             logger.warning("Already disconnected!")
 
+
     def _button_ping(self):
         if self.connected:
             self.network_client.ping()
 
+
+    def _button_view(self):
+        txt_map = {
+            True: "LOGS",
+            False: "VIDEO"
+        }
+        
+        self.logs_view = not self.logs_view
+        self.window["_multiline_log"].update(visible=self.logs_view)
+        self.window['_image_stream'].update(visible=not self.logs_view)
+        self.window["_button_view"].update("View: " + txt_map[self.logs_view])
+        if self.logs_view:
+            self.video_stream.disable()
+        else:
+            self.video_stream.enable()
+
     def update_op_values(self):
-        while not self.controller.log_queue.empty():
-            record = self.controller.log_queue.get()
-            self.window['_multiline_log'].update(record+"\n", append=True)
+        if self.logs_view:
+            while not self.controller.log_queue.empty():
+                record = self.controller.log_queue.get()
+                self.window['_multiline_log'].update(record+"\n", append=True)
+        else:
+            self.window['_image_stream'].update(data=self.video_stream.get_frame())
+
         self.robot_state = self.network_client.get_status()
         self.window["_text_mode"].update("M: " + Mode._VALUES_TO_NAMES[self.robot_state.mode])
         if self.robot_state.mode == Mode.WALK:
@@ -142,14 +179,16 @@ def run(self):
             if self.connected:
                 self.update_op_values()
             self.window.refresh()
-            time.sleep(0.05)
+            time.sleep(0.01)
 
 
     def refresh(self):
         self.window.refresh()
 
+
     def read(self):
         return window.read()
 
+
     def close(self):
         window.close()

steam_deck/video_stream.py

@@ -0,0 +1,48 @@
+import cv2
+import urllib.request
+import numpy as np
+import threading
+import logging
+
+logger = logging.getLogger(__name__)
+
+class VideoStream:
+
+    def __init__(self, addr: str): # 'http://192.168.0.6:9999/stream'
+        self.addr = addr
+        self.enabled = False
+        self._frame = b''
+
+
+    def _receive_video(self):
+        stream = urllib.request.urlopen(self.addr)
+        bin_data = b''
+        timeout = 1
+        while self.enabled:
+            bin_data += stream.read(1024)
+            a = bin_data.find(b'\xff\xd8')
+            b = bin_data.find(b'\xff\xd9')
+            if a != -1 and b != -1:
+                jpg = bin_data[a:b+2]
+                bin_data = bin_data[b+2:]
+                img_data = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_COLOR)
+                self._frame = cv2.imencode('.ppm', img_data)[1].tobytes()
+
+    def enable(self):
+        if self.enabled:
+            logger.warning("Unable to enable video stream. Already enabled")
+        else:
+            self.video_thread = threading.Thread(target=self._receive_video)
+            self.video_thread.start()
+            self.enabled = True  
+
+    def disable(self):
+        if not self.enabled:
+            logger.warning("Unable to disable video stream. Already disabled")
+        else:
+            self.enabled = False
+            self.video_thread.join() # TODO what if it hangs? 
+        
+    def get_frame(self):
+        return self._frame
+