Skip to content

Commit

Permalink
feat: nestri-server resiliency, ICE trickling and WebSocket as initia…
Browse files Browse the repository at this point in the history 
…tor connection (nestrilabs#137)

Using the same WebRTC connection as DataChannel inputs, though switched
it to WHIP etc.

Draft since I'm still looking into some stuff.

---------

Co-authored-by: DatCaptainHorse <[email protected]>
  • Loading branch information
@DatCaptainHorse
DatCaptainHorse and DatCaptainHorse committed Dec 8, 2024
1 parent 14eb24d commit aba8ecf
Show file tree
Hide file tree
Showing 27 changed files with 1,827 additions and 1,807 deletions.
952 changes: 94 additions & 858 deletions Cargo.lock
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ rust-version = "1.80"

[workspace.dependencies]
gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", version = "0.24.0" }
gst-app = { package = "gstreamer-app", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", version = "0.24.0" }
3 changes: 2 additions & 1 deletion Containerfile.runner
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ WORKDIR /builder/

# Grab build and rust packages #
RUN pacman -Syu --noconfirm meson pkgconf cmake git gcc make rustup \
gstreamer gst-plugins-base gst-plugins-good gst-plugin-rswebrtc
gstreamer gst-plugins-base gst-plugins-good

# Setup stable rust toolchain #
RUN rustup default stable
Expand Down Expand Up @@ -64,6 +64,7 @@ RUN pacman -Syu --noconfirm --needed \
gstreamer gst-plugins-base gst-plugins-good \
gst-plugin-va gst-plugins-bad gst-plugin-fmp4 \
gst-plugin-qsv gst-plugin-pipewire gst-plugin-rswebrtc \
gst-plugins-ugly gst-plugin-rsrtp \
# Audio packages
pipewire pipewire-pulse pipewire-alsa wireplumber \
# Other requirements
Expand Down
5 changes: 5 additions & 0 deletions apps/www/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"build.client": "vite build",
"build.preview": "vite build --ssr src/entry.preview.tsx",
"build.server": "vite build -c adapters/cloudflare-pages/vite.config.ts",
"deno:build.server": "vite build -c adapters/deno/vite.config.ts",
"build.types": "tsc --incremental --noEmit",
"deploy": "wrangler pages deploy ./dist",
"dev": "vite --mode ssr",
Expand Down Expand Up @@ -55,5 +56,9 @@
"vite": "5.3.5",
"vite-tsconfig-paths": "^4.2.1",
"wrangler": "^3.0.0"
},
"dependencies": {
"@types/pako": "^2.0.3",
"pako": "^2.1.0"
}
}
250 changes: 140 additions & 110 deletions apps/www/src/routes/play/[id]/index.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,117 +1,147 @@
import { useLocation } from "@builder.io/qwik-city";
import {useLocation} from "@builder.io/qwik-city";
import {Keyboard, Mouse, WebRTCStream} from "@nestri/input"
import { component$, useSignal, useVisibleTask$ } from "@builder.io/qwik";
import {component$, useSignal, useVisibleTask$} from "@builder.io/qwik";

export default component$(() => {
const id = useLocation().params.id;
const canvas = useSignal<HTMLCanvasElement>();

useVisibleTask$(({ track }) => {
track(() => canvas.value);

if (!canvas.value) return; // Ensure canvas is available

// Create video element and make it output to canvas (TODO: improve this)
let video = document.getElementById("webrtc-video-player");
if (!video) {
video = document.createElement("video");
video.id = "stream-video-player";
video.style.visibility = "hidden";
const webrtc = new WebRTCStream("https://nestri-relay.brumbas.se"); // or http://localhost:8088
webrtc.connect(id).then(() => {
const mediaStream = webrtc.getMediaStream();
console.log("Setting mediastream");
if (video && mediaStream) {
(video as HTMLVideoElement).srcObject = mediaStream;
const playbtn = document.createElement("button");
playbtn.style.position = "absolute";
playbtn.style.left = "50%";
playbtn.style.top = "50%";
playbtn.style.transform = "translateX(-50%) translateY(-50%)";
playbtn.style.width = "12rem";
playbtn.style.height = "6rem";
playbtn.style.borderRadius = "1rem";
playbtn.style.backgroundColor = "rgb(175, 50, 50)";
playbtn.style.color = "black";
playbtn.style.fontSize = "1.5em";
playbtn.textContent = "< Start >";

playbtn.onclick = () => {
playbtn.remove();
(video as HTMLVideoElement).play().then(() => {
if (canvas.value) {
canvas.value.width = (video as HTMLVideoElement).videoWidth;
canvas.value.height = (video as HTMLVideoElement).videoHeight;

const ctx = canvas.value.getContext("2d");
const renderer = () => {
if (ctx) {
ctx.drawImage((video as HTMLVideoElement), 0, 0);
requestAnimationFrame(renderer);
}
}
requestAnimationFrame(renderer);
}
});

document.addEventListener("pointerlockchange", (e) => {
if (!canvas.value) return; // Ensure canvas is available
// @ts-ignore
if (document.pointerLockElement && !window.nestrimouse && !window.nestrikeyboard) {
// @ts-ignore
window.nestrimouse = new Mouse({canvas: canvas.value, webrtc}, false); //< TODO: Make absolute mode toggleable, for now feels better?
// @ts-ignore
window.nestrikeyboard = new Keyboard({canvas: canvas.value, webrtc});
// @ts-ignore
} else if (!document.pointerLockElement && window.nestrimouse && window.nestrikeyboard) {
// @ts-ignore
window.nestrimouse.dispose();
// @ts-ignore
window.nestrimouse = undefined;
// @ts-ignore
window.nestrikeyboard.dispose();
// @ts-ignore
window.nestrikeyboard = undefined;
}
});
};

document.body.append(playbtn);
const id = useLocation().params.id;
const canvas = useSignal<HTMLCanvasElement>();

useVisibleTask$(({track}) => {
track(() => canvas.value);

if (!canvas.value) return; // Ensure canvas is available

// Create video element and make it output to canvas (TODO: improve this)
let video = document.getElementById("webrtc-video-player");
if (!video) {
video = document.createElement("video");
video.id = "stream-video-player";
video.style.visibility = "hidden";
const webrtc = new WebRTCStream("http://localhost:8088", id, (mediaStream) => {
if (video && mediaStream && (video as HTMLVideoElement).srcObject === null) {
console.log("Setting mediastream");
(video as HTMLVideoElement).srcObject = mediaStream;

// @ts-ignore
window.hasstream = true;
// @ts-ignore
window.roomOfflineElement?.remove();

const playbtn = document.createElement("button");
playbtn.style.position = "absolute";
playbtn.style.left = "50%";
playbtn.style.top = "50%";
playbtn.style.transform = "translateX(-50%) translateY(-50%)";
playbtn.style.width = "12rem";
playbtn.style.height = "6rem";
playbtn.style.borderRadius = "1rem";
playbtn.style.backgroundColor = "rgb(175, 50, 50)";
playbtn.style.color = "black";
playbtn.style.fontSize = "1.5em";
playbtn.textContent = "< Start >";

playbtn.onclick = () => {
playbtn.remove();
(video as HTMLVideoElement).play().then(() => {
if (canvas.value) {
canvas.value.width = (video as HTMLVideoElement).videoWidth;
canvas.value.height = (video as HTMLVideoElement).videoHeight;

const ctx = canvas.value.getContext("2d");
const renderer = () => {
// @ts-ignore
if (ctx && window.hasstream) {
ctx.drawImage((video as HTMLVideoElement), 0, 0);
(video as HTMLVideoElement).requestVideoFrameCallback(renderer);
}
}
(video as HTMLVideoElement).requestVideoFrameCallback(renderer);
}
});

document.addEventListener("pointerlockchange", (e) => {
if (!canvas.value) return; // Ensure canvas is available
// @ts-ignore
if (document.pointerLockElement && !window.nestrimouse && !window.nestrikeyboard) {
// @ts-ignore
window.nestrimouse = new Mouse({canvas: canvas.value, webrtc});
// @ts-ignore
window.nestrikeyboard = new Keyboard({canvas: canvas.value, webrtc});
// @ts-ignore
} else if (!document.pointerLockElement && window.nestrimouse && window.nestrikeyboard) {
// @ts-ignore
window.nestrimouse.dispose();
// @ts-ignore
window.nestrimouse = undefined;
// @ts-ignore
window.nestrikeyboard.dispose();
// @ts-ignore
window.nestrikeyboard = undefined;
}
});
};
document.body.append(playbtn);
} else if (mediaStream === null) {
console.log("MediaStream is null, Room is offline");
// Add a message to the screen
const offline = document.createElement("div");
offline.style.position = "absolute";
offline.style.left = "50%";
offline.style.top = "50%";
offline.style.transform = "translateX(-50%) translateY(-50%)";
offline.style.width = "auto";
offline.style.height = "auto";
offline.style.color = "lightgray";
offline.style.fontSize = "2em";
offline.textContent = "Offline";
document.body.append(offline);
// @ts-ignore
window.roomOfflineElement = offline;
// @ts-ignore
window.hasstream = false;
// Clear canvas if it has been set
if (canvas.value) {
const ctx = canvas.value.getContext("2d");
if (ctx) ctx.clearRect(0, 0, canvas.value.width, canvas.value.height);
}
}
})

return (
<canvas
ref={canvas}
onClick$={async () => {
if (canvas.value) {
// await element.value.requestFullscreen()
// Do not use - unadjustedMovement: true - breaks input on linux
canvas.value.requestPointerLock();
}
}}
//TODO: go full screen, then lock on "landscape" screen-orientation on mobile
class="aspect-video h-full w-full object-contain max-h-screen"/>
)
});
}
})

return (
<canvas
ref={canvas}
onClick$={async () => {
// @ts-ignore
if (canvas.value && window.hasstream) {
// await element.value.requestFullscreen()
// Do not use - unadjustedMovement: true - breaks input on linux
canvas.value.requestPointerLock();
}
}}
//TODO: go full screen, then lock on "landscape" screen-orientation on mobile
class="aspect-video h-full w-full object-contain max-h-screen"/>
)
})

{/**
.spinningCircleInner_b6db20 {
transform: rotate(280deg);
.spinningCircleInner_b6db20 {
transform: rotate(280deg);
}
.inner_b6db20 {
position: relative;
display: inline-flex;
align-items: center;
justify-content: center;
width: 32px;
height: 32px;
contain: paint;
} */
}

{/* <div class="loadingPopout_a8c724" role="dialog" tabindex="-1" aria-modal="true"><div class="spinner_b6db20 spinningCircle_b6db20" role="img" aria-label="Loading"><div class="spinningCircleInner_b6db20 inner_b6db20"><svg class="circular_b6db20" viewBox="25 25 50 50"><circle class="path_b6db20 path3_b6db20" cx="50" cy="50" r="20"></circle><circle class="path_b6db20 path2_b6db20" cx="50" cy="50" r="20"></circle><circle class="path_b6db20" cx="50" cy="50" r="20"></circle></svg></div></div></div> */
}
.inner_b6db20 {
position: relative;
display: inline-flex;
align-items: center;
justify-content: center;
width: 32px;
height: 32px;
contain: paint;
} */}

{/* <div class="loadingPopout_a8c724" role="dialog" tabindex="-1" aria-modal="true"><div class="spinner_b6db20 spinningCircle_b6db20" role="img" aria-label="Loading"><div class="spinningCircleInner_b6db20 inner_b6db20"><svg class="circular_b6db20" viewBox="25 25 50 50"><circle class="path_b6db20 path3_b6db20" cx="50" cy="50" r="20"></circle><circle class="path_b6db20 path2_b6db20" cx="50" cy="50" r="20"></circle><circle class="path_b6db20" cx="50" cy="50" r="20"></circle></svg></div></div></div> */ }
// .loadingPopout_a8c724 {
// background-color: var(--background-secondary);
// display: flex;
Expand Down Expand Up @@ -152,8 +182,8 @@ circle[Attributes Style] {
user agent stylesheet
:not(svg) {
transform-origin: 0px 0px;
} */}

} */
}


// .path2_b6db20 {
Expand Down Expand Up @@ -209,21 +239,21 @@ user agent stylesheet
// InputMessage::MouseMove { x, y } => {
// let mut last_move = state.last_mouse_move.lock().unwrap();
// let now = Instant::now();

// // Only process if coordinates are different or enough time has passed
// if (last_move.0 != x || last_move.1 != y) &&
// (now.duration_since(last_move.2).as_millis() > 16) { // ~60fps

// println!("Mouse moved to x: {}, y: {}", x, y);

// let structure = gst::Structure::builder("MouseMoveRelative")
// .field("pointer_x", x as f64)
// .field("pointer_y", y as f64)
// .build();

// let event = gst::event::CustomUpstream::new(structure);
// pipeline.send_event(event);

// // Update last position and time
// *last_move = (x, y, now);
// }
Expand Down
Binary file modified bun.lockb
View file
Binary file not shown.
9 changes: 7 additions & 2 deletions packages/input/src/keyboard.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {type Input} from "./types"
import {keyCodeToLinuxEventCode} from "./codes"
import { WebRTCStream } from "./webrtc-stream";
import {WebRTCStream, MessageInput, encodeMessage} from "./webrtc-stream";

interface Props {
webrtc: WebRTCStream;
Expand Down Expand Up @@ -67,7 +67,12 @@ export class Keyboard {
return;

const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
this.wrtc.sendData(JSON.stringify({...data, type} as Input));
const dataString = JSON.stringify({...data, type} as Input);
const message: MessageInput = {
payload_type: "input",
data: dataString,
};
this.wrtc.sendBinary(encodeMessage(message));
};
}

Expand Down
Loading

0 comments on commit aba8ecf

Please sign in to comment.