RealityTalk

package.json

@@ -3,12 +3,35 @@
  "version": "0.1.0",
  "private": true,
  "dependencies": {
+ "@mediapipe/camera_utils": "^0.3.1640029074",
+ "@mediapipe/control_utils": "^0.6.1629159505",
+ "@mediapipe/drawing_utils": "^0.3.1620248257",
+ "@mediapipe/hands": "^0.4.1646424915",
+ "@reduxjs/toolkit": "^1.8.0",
  "@testing-library/jest-dom": "^5.16.2",
  "@testing-library/react": "^12.1.4",
  "@testing-library/user-event": "^13.5.0",
+ "cors": "^2.8.5",
+ "duck-duck-scrape": "^2.2.1",
+ "express": "^4.17.3",
+ "konva": "^8.3.4",
+ "python-shell": "^3.0.1",
  "react": "^17.0.2",
  "react-dom": "^17.0.2",
+ "react-iframe": "^1.8.0",
+ "react-konva": "^17.0.2-5",
+ "react-konva-utils": "^0.2.0",
+ "react-papaparse": "^4.0.2",
+ "react-redux": "^7.2.6",
  "react-scripts": "5.0.0",
+ "react-spring": "^9.4.4",
+ "react-webcam": "^7.0.0",
+ "redux": "^4.1.2",
+ "socket.io": "^4.4.1",
+ "socket.io-client": "^4.4.1",
+ "spacy": "^0.0.4",
+ "tracking": "^1.1.3",
+ "use-image": "^1.0.10",
  "web-vitals": "^2.1.4"
  },
  "scripts": {
@@ -34,5 +57,8 @@
  "last 1 firefox version",
  "last 1 safari version"
  ]
+ },
+ "devDependencies": {
+ "@redux-devtools/core": "^3.11.0"
  }
 }

public/index.html

@@ -9,7 +9,7 @@
  name="description"
  content="Web site created using create-react-app"
  />
- <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
+<!-- <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />-->
  <!--
  manifest.json provides metadata used when your web app is installed on a
  user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/

public/manifest.json

@@ -1,23 +1,6 @@
 {
  "short_name": "React App",
  "name": "Create React App Sample",
- "icons": [
- {
- "src": "favicon.ico",
- "sizes": "64x64 32x32 24x24 16x16",
- "type": "image/x-icon"
- },
- {
- "src": "logo192.png",
- "type": "image/png",
- "sizes": "192x192"
- },
- {
- "src": "logo512.png",
- "type": "image/png",
- "sizes": "512x512"
- }
- ],
  "start_url": ".",
  "display": "standalone",
  "theme_color": "#000000",

server.js

@@ -0,0 +1,48 @@
+const {PythonShell} = require('python-shell')
+const pyshell = new PythonShell('spacy-listen.py')
+const express = require('express')
+const http = require('http')
+const app = express()
+const socketio = require('socket.io');
+const server = http.createServer(app)
+const cors = require('cors');
+const DDG = require('duck-duck-scrape')
+
+app.use(cors()); // add this line
+
+const io = socketio(server,{
+ cors: {
+ origin: '*',
+ methods: ['GET', 'POST']
+ }
+});
+
+
+server.listen(4000, () => {
+ console.log('Spacy server listening on 4000')
+})
+
+io.on('connection', (socket) => {
+ console.log('connection start')
+ socket.on('message', (message) => {
+ pyshell.send(message)
+ })
+
+ pyshell.on('message', (message) => {
+ socket.emit('message', message)
+ })
+
+ socket.on('search', (keyword) => {
+ keyword = keyword.replace(' ', '+')
+ const searchResults = DDG.search(keyword, {
+ safeSearch: DDG.SafeSearchType.STRICT
+ }).then(function(res) {
+ console.log(res)
+ socket.emit('search', res)
+ })
+ })
+
+ socket.on('disconnect', () => {
+ console.log('connection end')
+ })
+})

spacy-listen.py

@@ -0,0 +1,28 @@
+import json
+import spacy
+import pytextrank
+nlp = spacy.load("en_core_web_sm")
+nlp.add_pipe("merge_noun_chunks")
+nlp.add_pipe("merge_entities")
+nlp.add_pipe("textrank")
+
+while True:
+ text = input()
+ doc = nlp(text)
+
+ keywords = {}
+ for phrase in doc._.phrases:
+ keywords[phrase.text] = phrase.rank
+
+ res = { "tokens": [] }
+ for token in doc:
+ rank = keywords.get(token.text, 0)
+ res["tokens"].append({
+ "text": token.text,
+ "tag": token.tag_,
+ "is_stop": token.is_stop,
+ "ent_type": token.ent_type_,
+ "keyword_rank": rank,
+ })
+
+ print(json.dumps(res))

src/App.js

@@ -1,25 +1,61 @@
-import logo from './logo.svg';
 import './App.css';
+import HandsRecognition from "./Components/hands";
+import KonvaLayer from "./Components/konva";
+import {Speech, TestSpeech} from "./Components/speech";
+import ColorTracking from "./Components/color";
+import {useSelector} from "react-redux";
+import {selectObjectMode} from "./app/colorReducer";
+import {usePapaParse} from 'react-papaparse';
+
 
 function App() {
- return (
- <div className="App">
- <header className="App-header">
- <img src={logo} className="App-logo" alt="logo" />
- <p>
- Edit <code>src/App.js</code> and save to reload.
- </p>
- <a
- className="App-link"
- href="https://reactjs.org"
- target="_blank"
- rel="noopener noreferrer"
- >
- Learn React
- </a>
- </header>
- </div>
- );
+ let objectMode = useSelector(selectObjectMode)
+
+ return (
+ <div className="App">
+ <>
+ {/*<ReadString/>*/}
+ {objectMode ? <ColorTracking/> : <></>}
+ {/*<Speech/>*/}
+ <TestSpeech/>
+ <HandsRecognition/>
+ <KonvaLayer/>
+ </>
+
+ </div>
+ );
+}
+
+
+function ReadString() {
+ const {readString} = usePapaParse();
+
+ const handleReadString = () => {
+ const csvString = `Column 1,Column 2,Column 3,Column 4
+1-1,1-2,1-3,1-4
+2-1,2-2,2-3,2-4
+3-1,3-2,3-3,3-4
+4,5,6,7`;
+
+ readString(csvString, {
+ worker: true,
+ header: true,
+ complete: (results) => {
+ console.log('---------------------------');
+ let object = new Object();
+ results.data.forEach((ele) => {
+ let name = ele['Column 1'];
+ let val = ele['Column 2'];
+ Object.assign(object, {[name]: val})
+ })
+ console.log(object)
+ console.log('---------------------------');
+ console.log(object['1-1'])
+ },
+ });
+ };
+
+ return <button onClick={() => handleReadString()}>readString</button>;
 }
 
 export default App;

src/Components/KonvaElements/embeddedscreen.js

@@ -0,0 +1,39 @@
+import {useEffect, useRef, useState} from "react";
+import {Group, Label, Tag, Text} from "react-konva";
+import {Html} from "react-konva-utils";
+import Iframe from "react-iframe";
+
+export const EmbeddedScreen = (props) => {
+ const textRef = useRef(null)
+ const [x, setX] = useState(0)
+ // "http://www.youtube.com/embed/xDMP3i36naA"
+
+ useEffect(() => {
+ setX(textRef.current.textWidth + 15)
+ }, [textRef])
+
+ return (
+ <Group draggable={true} width={props.width} height={props.height} x={props.x} y={props.y} opacity={0.75}>
+ <Html>
+ <Iframe url={props.url}
+ width={props.width}
+ height={props.height}
+ display="initial"
+ />
+ </Html>
+ <Label x={(props.width / 2) - (x / 2)} y={-60}>
+ <Tag fill={'#62A6BF'}/>
+ <Text
+ ref={textRef}
+ text={props.text}
+ fontFamily={'Helvetica'}
+ fontSize={25}
+ padding={15}
+ fill={'white'}
+ align={'center'}
+ verticalAlign={'middle'}
+ />
+ </Label>
+ </Group>)
+
+}

src/Components/KonvaElements/groupwrapper.js

@@ -0,0 +1,58 @@
+import {useEffect, useRef, useState} from "react";
+import Konva from "konva";
+import {Group, Label, Tag, Text} from "react-konva";
+
+export function withGroup(Wrapped) {
+ return function (props) {
+ const groupRef = useRef(null)
+ const [xp, setXp] = useState(0)
+ const [yp, setYp] = useState(0)
+ const {x, y, timeout, following, ...passThroughProps} = props;
+
+ useEffect(() => {
+ setXp(x)
+ setYp(y)
+
+ groupRef.current.to({
+ // scaleX: 1,
+ // scaleY: 1,
+ opacity: 0.75,
+ easing: Konva.Easings.BackEaseInOut,
+ duration: 0.4
+ });
+
+ setTimeout(() => {
+ if (groupRef.current !== null) {
+ groupRef.current.to({
+ scaleX: 0,
+ scaleY: 0,
+ opacity: 0,
+ easing: Konva.Easings.BackEaseInOut,
+ duration: 0.4
+ })
+ setTimeout(() => groupRef.current.destroy(), 400)
+ }
+ }, timeout);
+
+ }, [groupRef])
+
+ useEffect(() => {
+ if (following) {
+ setXp(x)
+ setYp(y)
+ }
+ }, [x])
+
+ return (
+ <Group
+ ref={groupRef}
+ draggable={true} opacity={0}
+ // scale={groupRef.current === null ? {x: 0, y: 0} : {x: 1, y: 1}}
+ x={following ? x : xp} y={following ? y : yp}
+ >
+ <Wrapped {...passThroughProps} />
+ </Group>
+ );
+ }
+}
+

src/Components/KonvaElements/layerimage.js

@@ -0,0 +1,33 @@
+// URL Image Component
+import useImage from "use-image";
+import {useEffect, useRef, useState} from "react";
+import {Image} from "react-konva";
+import {withGroup} from "./groupwrapper";
+import {Textual} from "./textual";
+
+export const LayerImage = (props) => {
+ // 'https://konvajs.org/assets/lion.png'
+ const [image, status] = useImage(props.url, 'anonymous');
+ const imageRef = useRef(null);
+ const [scaleX, setScaleX] = useState(0);
+ const scaleProps = props.scale
+
+ useEffect(() => {
+ if (imageRef.current !== null && status === 'loaded') {
+ // const height = imageRef.current.attrs.image.height;
+ const width = imageRef.current.attrs.image.width;
+ // const scaleHeight = 0.15 * window.innerHeight;
+ const scaleWidth = 0.15 * window.innerWidth;
+ setScaleX(scaleWidth / width);
+ // setScaleY(scaleHeight / height);
+ }
+ }, [imageRef, status])
+
+ useEffect(() => {
+ setScaleX()
+ }, [scaleProps])
+
+ return <Image ref={imageRef} image={image} x={props.x} y={props.y}
+ scale={{x: scaleX, y: scaleX}}/>;
+};
+

src/Components/KonvaElements/textual.js

@@ -0,0 +1,21 @@
+import {Label, Tag, Text} from "react-konva";
+import {withGroup} from "./groupwrapper";
+
+export function Textual(props) {
+ return (
+ <Label>
+ <Tag fill={'#62A6BF'}/>
+ <Text text={props.text}
+ fontFamily={'Helvetica'}
+ fontSize={35}
+ padding={20}
+ fill={'white'}
+ align={'center'}
+ verticalAlign={'middle'}
+ />
+ </Label>
+ );
+}
+
+
+