Skip to content

Commit

Permalink
support screen recording
Browse files Browse the repository at this point in the history
  • Loading branch information
abi committed Mar 14, 2024
1 parent b69edb7 commit c08cf0a
Show file tree
Hide file tree
Showing 7 changed files with 247 additions and 29 deletions.
26 changes: 13 additions & 13 deletions backend/poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion frontend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@
"tailwind-merge": "^2.0.0",
"tailwindcss-animate": "^1.0.7",
"thememirror": "^2.0.1",
"vite-plugin-checker": "^0.6.2"
"vite-plugin-checker": "^0.6.2",
"webm-duration-fix": "^1.0.4"
},
"devDependencies": {
"@types/node": "^20.9.0",
Expand Down
52 changes: 37 additions & 15 deletions frontend/src/components/ImageUpload.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import { useDropzone } from "react-dropzone";
import { toast } from "react-hot-toast";
import { URLS } from "../urls";
import { Badge } from "./ui/badge";
import ScreenRecorder from "./recording/ScreenRecorder";
import { ScreenRecorderState } from "../types";

const baseStyle = {
flex: 1,
Expand Down Expand Up @@ -61,16 +63,23 @@ interface Props {

function ImageUpload({ setReferenceImages }: Props) {
const [files, setFiles] = useState<FileWithPreview[]>([]);
// TODO: Switch to Zustand
const [screenRecorderState, setScreenRecorderState] =
useState<ScreenRecorderState>(ScreenRecorderState.INITIAL);

const { getRootProps, getInputProps, isFocused, isDragAccept, isDragReject } =
useDropzone({
maxFiles: 1,
maxSize: 1024 * 1024 * 20, // 20 MB
accept: {
// Image formats
"image/png": [".png"],
"image/jpeg": [".jpeg"],
"image/jpg": [".jpg"],
// Video formats
"video/quicktime": [".mov"],
"video/mp4": [".mp4"],
"video/webm": [".webm"],
},
onDrop: (acceptedFiles) => {
// Set up the preview thumbnail images
Expand Down Expand Up @@ -154,21 +163,34 @@ function ImageUpload({ setReferenceImages }: Props) {

return (
<section className="container">
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any */}
<div {...getRootProps({ style: style as any })}>
<input {...getInputProps()} />
<p className="text-slate-700 text-lg">
Drag & drop a screenshot here, <br />
or click to upload
</p>
</div>
<div className="text-center text-sm text-slate-800 mt-4">
<Badge>New!</Badge> Upload a screen recording in .mp4 or .mov format to
clone a whole app (experimental).{" "}
<a className="underline" href={URLS["intro-to-video"]} target="_blank">
Learn more.
</a>
</div>
{screenRecorderState === ScreenRecorderState.INITIAL && (
/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
<div {...getRootProps({ style: style as any })}>
<input {...getInputProps()} />
<p className="text-slate-700 text-lg">
Drag & drop a screenshot here, <br />
or click to upload
</p>
</div>
)}
{screenRecorderState === ScreenRecorderState.INITIAL && (
<div className="text-center text-sm text-slate-800 mt-4">
<Badge>New!</Badge> Upload a screen recording (.mp4, .mov) or record
your screen to clone a whole app (experimental).{" "}
<a
className="underline"
href={URLS["intro-to-video"]}
target="_blank"
>
Learn more.
</a>
</div>
)}
<ScreenRecorder
screenRecorderState={screenRecorderState}
setScreenRecorderState={setScreenRecorderState}
generateCode={setReferenceImages}
/>
</section>
);
}
Expand Down
115 changes: 115 additions & 0 deletions frontend/src/components/recording/ScreenRecorder.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import { useState } from "react";
import { Button } from "../ui/button";
import { ScreenRecorderState } from "../../types";
import { blobToBase64DataUrl } from "./utils";
import fixWebmDuration from "webm-duration-fix";
import toast from "react-hot-toast";

interface Props {
screenRecorderState: ScreenRecorderState;
setScreenRecorderState: (state: ScreenRecorderState) => void;
generateCode: (
referenceImages: string[],
inputMode: "image" | "video"
) => void;
}

function ScreenRecorder({
screenRecorderState,
setScreenRecorderState,
generateCode,
}: Props) {
const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(
null
);
const [screenRecordingDataUrl, setScreenRecordingDataUrl] = useState<
string | null
>(null);

const startScreenRecording = async () => {
try {
// Get the screen recording stream
const stream = await navigator.mediaDevices.getDisplayMedia({
video: true,
audio: { echoCancellation: true },
});

// TODO: Test across different browsers
// Create the media recorder
const options = { mimeType: "video/webm" };
const mediaRecorder = new MediaRecorder(stream, options);
setMediaRecorder(mediaRecorder);

const chunks: BlobPart[] = [];

// Accumalate chunks as data is available
mediaRecorder.ondataavailable = (e: BlobEvent) => chunks.push(e.data);

// When media recorder is stopped, create a data URL
mediaRecorder.onstop = async () => {
// TODO: Do I need to fix duration if it's not a webm?
const completeBlob = await fixWebmDuration(
new Blob(chunks, {
type: options.mimeType,
})
);

const dataUrl = await blobToBase64DataUrl(completeBlob);
setScreenRecordingDataUrl(dataUrl);
setScreenRecorderState(ScreenRecorderState.FINISHED);
};

// Start recording
mediaRecorder.start();
setScreenRecorderState(ScreenRecorderState.RECORDING);
} catch (error) {
toast.error("Could not start screen recording");
throw error;
}
};

const stopScreenRecording = () => {
if (mediaRecorder) {
mediaRecorder.stop();
setMediaRecorder(null);
}
};

const kickoffGeneration = () => {
if (screenRecordingDataUrl) {
generateCode([screenRecordingDataUrl], "video");
} else {
toast.error("Screen recording does not exist. Please try again.");
throw new Error("No screen recording data url");
}
};

return (
<div className="flex items-center justify-center my-3">
{screenRecorderState === ScreenRecorderState.INITIAL && (
<Button onClick={startScreenRecording}>Record Screen</Button>
)}

{screenRecorderState === ScreenRecorderState.RECORDING && (
<div className="flex items-center flex-col gap-y-4">
<div className="flex items-center mr-2 text-xl gap-x-1">
<span className="block h-10 w-10 bg-red-600 rounded-full mr-1 animate-pulse"></span>
<span>Recording...</span>
</div>
<Button onClick={stopScreenRecording}>Finish Recording</Button>
</div>
)}

{screenRecorderState === ScreenRecorderState.FINISHED && (
<div className="flex items-center flex-col gap-y-4">
<div className="flex items-center mr-2 text-xl gap-x-1">
<span>Screen Recording Captured.</span>
</div>
<Button onClick={kickoffGeneration}>Generate</Button>
</div>
)}
</div>
);
}

export default ScreenRecorder;
31 changes: 31 additions & 0 deletions frontend/src/components/recording/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
export function downloadBlob(blob: Blob) {
// Create a URL for the blob object
const videoURL = URL.createObjectURL(blob);

// Create a temporary anchor element and trigger the download
const a = document.createElement("a");
a.href = videoURL;
a.download = "recording.webm";
document.body.appendChild(a);
a.click();
document.body.removeChild(a);

// Clear object URL
URL.revokeObjectURL(videoURL);
}

export function blobToBase64DataUrl(blob: Blob): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onloadend = () => {
if (reader.result) {
resolve(reader.result as string);
} else {
reject(new Error("FileReader did not return a result."));
}
};
reader.onerror = () =>
reject(new Error("FileReader encountered an error."));
reader.readAsDataURL(blob);
});
}
6 changes: 6 additions & 0 deletions frontend/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ export enum AppState {
CODE_READY = "CODE_READY",
}

export enum ScreenRecorderState {
INITIAL = "initial",
RECORDING = "recording",
FINISHED = "finished",
}

export interface CodeGenerationParams {
generationType: "create" | "update";
inputMode: "image" | "video";
Expand Down
Loading

0 comments on commit c08cf0a

Please sign in to comment.