Viewer refactor with state sync (nerfstudio-project#212)

* initial commit for viewer refactor

* updating name change to nerfactory

* adding headers

* making the rendering fix which broke from a merge

* things are starting to work. removing old commands now

* removed prints

* cleanup

* updating state works now even for nested dictionaries

* updating state in redux. listening for state changes. trying to draw three.js now

* added drawing functionality. need to fix translation of frustums and image planes

* images and frustums drawn correctly

* fixing lint

* github actions

* back to npm

* fixing eslint

* remove package.json from root

* new favicon

* some viewer cleanup

* notebook

* fixed camera drawings speed to avoid redraw

* trying to get Leva to update

* quick fix

* synced config panel

* adding notebook

* fixing output print error

.github/workflows/viewer_client_app.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Install Node.js
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: 17.8.0
 
@@ -54,4 +54,4 @@ jobs:
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./nerfactory/viewer/app/build
+          publish_dir: ./nerfactory/viewer/app/build

.gitignore

@@ -121,6 +121,7 @@ celerybeat.pid
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/

.prettierrc.js

@@ -0,0 +1,6 @@
+module.exports = {
+    trailingComma: 'all',
+    arrowParens: 'always',
+    singleQuote: true,
+    jsxSingleQuote: false,
+  };

.vscode/launch.json

@@ -4,6 +4,13 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Launch Chrome against localhost",
+      "request": "launch",
+      "type": "chrome",
+      "url": "http://localhost:4000",
+      "webRoot": "${workspaceFolder}/nerfactory/viewer/app"
+    },
     {
       "name": "Python: Current File",
       "type": "python",
@@ -25,23 +32,29 @@
       "request": "launch",
       "program": "scripts/run_train.py",
       "console": "integratedTerminal",
-      "args": ["machine.num_gpus=0"]
+      "args": [
+        "machine.num_gpus=0"
+      ]
     },
     {
       "name": "Python: run_train ngp (gpu)",
       "type": "python",
       "request": "launch",
       "program": "scripts/run_train.py",
       "console": "integratedTerminal",
-      "args": ["--config-name=graph_instant_ngp.yaml"]
+      "args": [
+        "--config-name=graph_instant_ngp.yaml"
+      ]
     },
     {
       "name": "Python: run_train mipnerf 360 (gpu)",
       "type": "python",
       "request": "launch",
       "program": "scripts/run_train.py",
       "console": "integratedTerminal",
-      "args": ["--config-name=graph_mipnerf_360.yaml"]
+      "args": [
+        "--config-name=graph_mipnerf_360.yaml"
+      ]
     },
     {
       "name": "Python: run_train nerfw (gpu)",
@@ -66,4 +79,4 @@
       ]
     }
   ]
-}
+}

.vscode/settings.json

@@ -8,6 +8,21 @@
     "**/temp*": true
   },
   "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "[javascript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[javascriptreact]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[typescriptreact]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "prettier.configPath": ".prettierrc.js",
+  "typescript.suggestionActions.enabled": false,
+  "javascript.suggestionActions.enabled": false,
   "[python]": {
     "editor.defaultFormatter": "ms-python.python",
     "editor.codeActionsOnSave": {
@@ -17,18 +32,34 @@
   "editor.formatOnSave": true,
   "python.envFile": "${workspaceFolder}/.env",
   "python.formatting.provider": "black",
-  "python.formatting.blackArgs": ["--line-length=120"],
+  "python.formatting.blackArgs": [
+    "--line-length=120"
+  ],
   "python.linting.pylintEnabled": true,
   "python.linting.flake8Enabled": false,
   "python.linting.enabled": true,
-  "python.sortImports.args": ["--src=${workspaceFolder}", "--profile", "black"],
+  "python.sortImports.args": [
+    "--src=${workspaceFolder}",
+    "--profile",
+    "black"
+  ],
   "python.testing.unittestEnabled": false,
   "python.testing.pytestEnabled": true,
   "python.linting.ignorePatterns": [
     "**/site-packages/**/*.py",
     ".vscode/*.py",
     "external/**/*.py"
   ],
+  "esbonio.sphinx.confDir": "",
+  // eslint extension options
+  "javascript.validate.enable": false,
+  "eslint.enable": true,
+  "eslint.validate": [
+    "javascript",
+    "javascriptreact",
+    "typescript",
+    "typescriptreact"
+  ],
   "C_Cpp.errorSquiggles": "Disabled",
   "files.associations": {
     "array": "cpp",
@@ -96,4 +127,4 @@
     "__node_handle": "cpp"
   },
   "python.analysis.typeCheckingMode": "basic"
-}
+}

docs/notebooks/index.rst

@@ -11,4 +11,5 @@ Components
     visualize_samplers.ipynb
     visualize_spatial_distortions.ipynb
     visualize_encoders.ipynb
+    visualize_viewer.ipynb
 

nerfactory/cameras/cameras.py

@@ -17,14 +17,16 @@
 """
 import base64
 from abc import abstractmethod
-from typing import Dict, Optional, Type
+from typing import Dict, Optional, Tuple, Type
 
 import cv2
+import numpy as np
 import torch
 from torch.nn.functional import normalize
 from torchtyping import TensorType
 
 from nerfactory.cameras.rays import RayBundle
+from nerfactory.utils.misc import is_not_none
 
 
 class Camera:
@@ -161,11 +163,15 @@ def get_camera_ray_bundle(self, device=None) -> RayBundle:
         return ray_bundle
 
     @abstractmethod
-    def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
+    def to_json(
+        self,
+        image: Optional[TensorType["image_height", "image_width", 2]] = None,
+        resize_shape: Optional[Tuple[int, int]] = None,
+    ) -> Dict:
         """Converts the camera to a json dictionary.
 
         Args:
-            image: An image that is encoded to a base64 string. Defaults to None.
+            image: An image in range [0, 1] that is encoded to a base64 string. Defaults to None.
 
         Returns:
             A JSON representation of the camera
@@ -295,7 +301,11 @@ def generate_rays(
 
         return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None])
 
-    def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
+    def to_json(
+        self,
+        image: Optional[TensorType["image_height", "image_width", 2]] = None,
+        resize_shape: Optional[Tuple[int, int]] = None,
+    ) -> Dict:
         json_ = {
             "type": "PinholeCamera",
             "cx": self.cx,
@@ -305,10 +315,11 @@ def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]]
             "camera_to_world": self.camera_to_world.tolist(),
             "camera_index": self.camera_index,
         }
-        if image:
-            # move image to cpu if not already
-            # TODO: move to numpy
-            data = cv2.imencode(".png", image[:, :, ::-1])[1].tobytes()
+        if is_not_none(image):
+            image_uint8 = (image * 255).detach().cpu().numpy().astype(np.uint8)
+            if resize_shape:
+                image_uint8 = cv2.resize(image_uint8, resize_shape)
+            data = cv2.imencode(".png", image_uint8)[1].tobytes()
             json_["image"] = str("data:image/png;base64," + base64.b64encode(data).decode("ascii"))
         return json_
 
@@ -346,9 +357,6 @@ def fx_index(cls):
     def fy_index(cls):
         return 2
 
-    def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
-        raise NotImplementedError
-
     @staticmethod
     def from_json(json_: Dict) -> "SimplePinholeCamera":  # pylint:disable=no-self-use
         raise NotImplementedError
@@ -441,7 +449,11 @@ def generate_rays(
         pixel_area = dx * dy
         return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None])
 
-    def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
+    def to_json(
+        self,
+        image: Optional[TensorType["image_height", "image_width", 2]] = None,
+        resize_shape: Optional[Tuple[int, int]] = None,
+    ) -> Dict:
         raise NotImplementedError
 
     @staticmethod

nerfactory/data/image_dataset.py

@@ -87,7 +87,7 @@ def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]:
         assert image.shape[2] in [3, 4], f"Image shape of {image.shape} is in correct."
         return image
 
-    def get_image(self, image_idx: int):
+    def get_image(self, image_idx: int) -> TensorType["image_height", "image_width", "num_channels"]:
         """Returns a 3 channel image."""
         image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32") / 255.0)
         if self.alpha_color is not None and image.shape[-1] == 4:

nerfactory/graphs/base.py

@@ -86,9 +86,6 @@ def __init__(
         self.populate_fields()
         self.populate_misc_modules()  # populate the modules
         self.callbacks = None
-        # variable for visualizer to fetch TODO(figure out if there is cleaner way to do this)
-        self.vis_outputs = None
-        self.default_output_name = None
         # to keep track of which device the nn.Module is on
         self.device_indicator_param = nn.Parameter(torch.empty(0))
 

nerfactory/viewer/app/.env.development

@@ -1,4 +1,5 @@
 BROWSER=none
 FAST_REFRESH=false
 HOST=localhost
-PORT=4000
+PORT=4000
+ESLINT_NO_DEV_ERRORS=true

nerfactory/viewer/app/.eslintrc.json

@@ -0,0 +1,33 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:react/recommended",
+        "airbnb",
+        "prettier"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaFeatures": {
+            "jsx": true
+        },
+        "ecmaVersion": "latest",
+        "sourceType": "module"
+    },
+    "plugins": [
+        "react",
+        "@typescript-eslint"
+    ],
+    "rules": {
+        "arrow-body-style": "off",
+        "camelcase": "off",
+        "import/prefer-default-export": "off",
+        "no-alert": "off",
+        "no-console": "off",
+        "prefer-destructuring": "off",
+        "react/destructuring-assignment": "off"
+    }
+}

nerfactory/viewer/app/package.json

@@ -4,28 +4,37 @@
   "version": "0.0.0",
   "private": true,
   "dependencies": {
+    "@reduxjs/toolkit": "^1.8.3",
     "@testing-library/jest-dom": "^5.16.4",
     "@testing-library/react": "^13.3.0",
     "@testing-library/user-event": "^14.2.0",
     "ccapture.js": "github:aceslowman/ccapture.js#npm-fix",
+    "classnames": "^2.3.1",
     "dat.gui": "^0.7.9",
+    "eslint-config-prettier": "^8.5.0",
     "leva": "^0.9.29",
     "msgpack-lite": "^0.1.26",
+    "prop-types": "^15.8.1",
     "react": "^18.1.0",
     "react-dom": "^18.1.0",
-    "react-scripts": "^5.0.1",
+    "react-redux": "^8.0.2",
+    "redux": "^4.2.0",
     "socket.io-client": "^4.5.1",
     "three": "^0.142.0",
     "three-wtm": "^1.0",
     "websocket": "^1.0.34",
-    "wwobjloader2": "^4.0"
+    "wwobjloader2": "^4.0",
+    "electron": "^19.0.4",
+    "electron-is-dev": "^2.0.0"
   },
   "scripts": {
     "start": "react-scripts start",
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
-    "electron": "electron ."
+    "electron": "electron .",
+    "lint": "eslint --ext .js,.jsx .",
+    "lint:fix": "eslint --fix --ext .js,.jsx ."
   },
   "eslintConfig": {
     "extends": "react-app"
@@ -47,10 +56,20 @@
   "license": "ISC",
   "description": "",
   "devDependencies": {
+    "@svgr/webpack": "^6.2.1",
+    "prettier": "2.7.1",
     "concurrently": "^7.2.1",
-    "electron": "^19.0.4",
-    "electron-is-dev": "^2.0.0",
+    "eslint": "^8.2.0",
+    "eslint-config-airbnb": "19.0.4",
+    "eslint-plugin-import": "^2.25.3",
+    "eslint-plugin-jsx-a11y": "^6.5.1",
+    "eslint-plugin-react": "^7.28.0",
+    "eslint-plugin-react-hooks": "^4.3.0",
+    "react-scripts": "^5.0.1",
     "typescript": "^4.7.3",
     "wait-on": "^6.0.1"
+  },
+  "resolutions": {
+    "@svgr/webpack": "$@svgr/webpack"
   }
 }

nerfactory/viewer/app/public/index.html

@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8" />
-    <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
+    <link rel="icon" href="%PUBLIC_URL%/favicon.png" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="theme-color" content="#000000" />
     <meta
@@ -24,7 +24,7 @@
       work correctly both with client-side routing and a non-root public URL.
       Learn how to configure a non-root public URL by running `npm run build`.
     -->
-    <title>nerfactory Viewer</title>
+    <title>Nerfviewer</title>
   </head>
 
   <body>