SAM2.1

SAM2.1 checkpoints + training code + Demo

demo/frontend/src/common/components/MobileFirstClickBanner.tsx

@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideoModal from '@/common/components/gallery/ChangeVideoModal';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {spacing} from '@/theme/tokens.stylex';
+import {ImageCopy} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {Button} from 'react-daisyui';
+
+const styles = stylex.create({
+  container: {
+    position: 'relative',
+    backgroundColor: '#000',
+    padding: spacing[5],
+    paddingVertical: spacing[6],
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing[4],
+  },
+});
+
+export default function MobileFirstClickBanner() {
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div className="flex text-white text-lg">
+        Click an object in the video to start
+      </div>
+      <div className="text-sm text-[#A7B3BF]">
+        <p>
+          You&apos;ll be able to use {DEMO_SHORT_NAME} to make fun edits to any
+          video by tracking objects and applying visual effects. To start, click
+          any object in the video.
+        </p>
+      </div>
+      <div className="flex items-center">
+        <ChangeVideoModal
+          videoGalleryModalTrigger={MobileVideoGalleryModalTrigger}
+          showUploadInGallery={true}
+        />
+      </div>
+    </div>
+  );
+}
+
+type MobileVideoGalleryModalTriggerProps = {
+  onClick: () => void;
+};
+
+function MobileVideoGalleryModalTrigger({
+  onClick,
+}: MobileVideoGalleryModalTriggerProps) {
+  return (
+    <Button
+      color="ghost"
+      startIcon={<ImageCopy size={20} />}
+      onClick={onClick}
+      className="text-white p-0">
+      Change video
+    </Button>
+  );
+}

demo/frontend/src/common/components/Tooltip.tsx

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {PropsWithChildren} from 'react';
+
+type Props = PropsWithChildren<{
+  className?: string;
+  message: string;
+  position?: 'left' | 'top' | 'right' | 'bottom';
+}>;
+
+/**
+ * This is a custom Tooltip component because React Daisy UI does not have an
+ * option to *only* show tooltip on large devices.
+ */
+export default function Tooltip({
+  children,
+  className = '',
+  message,
+  position = 'top',
+}: Props) {
+  return (
+    <div
+      className={`lg:tooltip tooltip-${position} ${className}`}
+      data-tip={message}>
+      {children}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/AddObjectButton.tsx

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {activeTrackletObjectIdAtom, labelTypeAtom} from '@/demo/atoms';
+import {Add} from '@carbon/icons-react';
+import {useSetAtom} from 'jotai';
+
+export default function AddObjectButton() {
+  const video = useVideo();
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+  const setLabelType = useSetAtom(labelTypeAtom);
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  async function addObject() {
+    enqueueMessage('addObjectClick');
+    const tracklet = await video?.createTracklet();
+    if (tracklet != null) {
+      setActiveTrackletId(tracklet.id);
+      setLabelType('positive');
+    }
+  }
+
+  return (
+    <div
+      onClick={addObject}
+      className="group flex justify-start mx-4 px-4 bg-transparent text-white !rounded-xl border-none cursor-pointer">
+      <div className="flex gap-6 items-center">
+        <div className=" group-hover:bg-graydark-700 border border-white relative h-12 w-12 md:w-20 md:h-20 shrink-0 rounded-lg flex items-center justify-center">
+          <Add size={36} className="group-hover:text-white text-gray-300" />
+        </div>
+        <div className="font-medium text-base">Add another object</div>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ClearAllPointsInVideoButton.tsx

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useRestartSession from '@/common/components/session/useRestartSession';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {isPlayingAtom, isStreamingAtom, labelTypeAtom} from '@/demo/atoms';
+import {Reset} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {useState} from 'react';
+import {Button, Loading} from 'react-daisyui';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+  },
+});
+
+type Props = {
+  onRestart: () => void;
+};
+
+export default function ClearAllPointsInVideoButton({onRestart}: Props) {
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setLabelType = useSetAtom(labelTypeAtom);
+  const {clearMessage} = useMessagesSnackbar();
+  const {restartSession} = useRestartSession();
+
+  const video = useVideo();
+
+  async function handleRestart() {
+    if (video === null) {
+      return;
+    }
+
+    setIsLoading(true);
+    if (isPlaying) {
+      video.pause();
+    }
+    if (isStreaming) {
+      await video.abortStreamMasks();
+    }
+    const isSuccessful = await video.clearPointsInVideo();
+    if (!isSuccessful) {
+      await restartSession();
+    }
+    video.frame = 0;
+    setLabelType('positive');
+    onRestart();
+    clearMessage();
+    setIsLoading(false);
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <Button
+        color="ghost"
+        onClick={handleRestart}
+        className="!px-4 !rounded-full font-medium text-white hover:bg-black"
+        startIcon={isLoading ? <Loading size="sm" /> : <Reset size={20} />}>
+        Start over
+      </Button>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/CloseSessionButton.tsx

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {ChevronRight} from '@carbon/icons-react';
+
+type Props = {
+  onSessionClose: () => void;
+};
+
+export default function CloseSessionButton({onSessionClose}: Props) {
+  const video = useVideo();
+
+  function handleCloseSession() {
+    video?.closeSession();
+    video?.logAnnotations();
+    onSessionClose();
+  }
+
+  return (
+    <PrimaryCTAButton onClick={handleCloseSession} endIcon={<ChevronRight />}>
+      Good to go
+    </PrimaryCTAButton>
+  );
+}

demo/frontend/src/common/components/annotations/FirstClickView.tsx

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideo from '@/common/components/gallery/ChangeVideoModal';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {useEffect, useRef} from 'react';
+
+export default function FirstClickView() {
+  const isFirstClickMessageShown = useRef(false);
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useEffect(() => {
+    if (!isFirstClickMessageShown.current) {
+      isFirstClickMessageShown.current = true;
+      enqueueMessage('firstClick');
+    }
+  }, [enqueueMessage]);
+
+  return (
+    <div className="w-full h-full flex flex-col p-8">
+      <div className="grow flex flex-col gap-6">
+        <h2 className="text-2xl">Click an object in the video to start</h2>
+        <p className="!text-gray-60">
+          You&apos;ll be able to use {DEMO_SHORT_NAME} to make fun edits to any
+          video by tracking objects and applying visual effects.
+        </p>
+        <p className="!text-gray-60">
+          To start, click any object in the video.
+        </p>
+      </div>
+      <div className="flex items-center">
+        <ChangeVideo />
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/LimitNotice.tsx

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {InformationFilled} from '@carbon/icons-react';
+
+export default function LimitNotice() {
+  return (
+    <div className="mt-6 gap-3 mx-6 flex items-center text-gray-400">
+      <div>
+        <InformationFilled size={32} />
+      </div>
+      <div className="text-sm leading-snug">
+        In this demo, you can track up to 3 objects, even though the SAM 2 model
+        does not have a limit.
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/MobileObjectsList.tsx

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ClearAllPointsInVideoButton from '@/common/components/annotations/ClearAllPointsInVideoButton';
+import ObjectThumbnail from '@/common/components/annotations/ObjectThumbnail';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import {activeTrackletObjectIdAtom, trackletObjectsAtom} from '@/demo/atoms';
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue, useSetAtom} from 'jotai';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    padding: spacing[5],
+    borderTop: '1px solid #DEE3E9',
+  },
+  trackletsContainer: {
+    flexGrow: 1,
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[5],
+  },
+});
+
+type Props = {
+  showActiveObject: () => void;
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileObjectsList({
+  showActiveObject,
+  onTabChange,
+}: Props) {
+  const tracklets = useAtomValue(trackletObjectsAtom);
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+
+  function handleSelectObject(tracklet: BaseTracklet) {
+    setActiveTrackletId(tracklet.id);
+    showActiveObject();
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.trackletsContainer)}>
+        {tracklets.map(tracklet => {
+          const {id, color, thumbnail} = tracklet;
+          return (
+            <ObjectThumbnail
+              key={id}
+              color={color}
+              thumbnail={thumbnail}
+              onClick={() => {
+                handleSelectObject(tracklet);
+              }}
+            />
+          );
+        })}
+      </div>
+      <ClearAllPointsInVideoButton
+        onRestart={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/MobileObjectsToolbar.tsx

@@ -0,0 +1,51 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import MobileObjectsToolbarHeader from '@/common/components/annotations/MobileObjectsToolbarHeader';
+import ObjectsToolbarBottomActions from '@/common/components/annotations/ObjectsToolbarBottomActions';
+import {getObjectLabel} from '@/common/components/annotations/ObjectUtils';
+import ToolbarObject from '@/common/components/annotations/ToolbarObject';
+import MobileFirstClickBanner from '@/common/components/MobileFirstClickBanner';
+import {activeTrackletObjectAtom, isFirstClickMadeAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileObjectsToolbar({onTabChange}: Props) {
+  const activeTracklet = useAtomValue(activeTrackletObjectAtom);
+  const isFirstClickMade = useAtomValue(isFirstClickMadeAtom);
+
+  if (!isFirstClickMade) {
+    return <MobileFirstClickBanner />;
+  }
+
+  return (
+    <div className="w-full">
+      <MobileObjectsToolbarHeader />
+      {activeTracklet != null && (
+        <ToolbarObject
+          label={getObjectLabel(activeTracklet)}
+          tracklet={activeTracklet}
+          isActive={true}
+          isMobile={true}
+        />
+      )}
+
+      <ObjectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/MobileObjectsToolbarHeader.tsx

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarProgressChip from '@/common/components/toolbar/ToolbarProgressChip';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function MobileObjectsToolbarHeader() {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  return (
+    <div className="w-full flex gap-4 items-center px-5 py-5">
+      <div className="grow text-sm text-white">
+        <ToolbarProgressChip />
+        {streamingState === 'full'
+          ? 'Review your selected objects across the video, and continue to edit if needed. Once everything looks good, press “Next” to continue.'
+          : isStreaming
+            ? 'Watch the video closely for any places where your objects aren’t tracked correctly. You can also stop tracking to make additional edits.'
+            : 'Edit your object selection with a few more clicks if needed. Press “Track objects” to track your objects throughout the video.'}
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectActions.tsx

@@ -0,0 +1,116 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PointsToggle from '@/common/components/annotations/PointsToggle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useReportError from '@/common/error/useReportError';
+import {
+  activeTrackletObjectIdAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+} from '@/demo/atoms';
+import {
+  AddFilled,
+  Select_02,
+  SubtractFilled,
+  TrashCan,
+} from '@carbon/icons-react';
+import {useAtom, useAtomValue} from 'jotai';
+import {useState} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = {
+  objectId: number;
+  active: boolean;
+};
+
+function CustomButton({className, ...props}: ButtonProps) {
+  return (
+    <Button
+      size="sm"
+      color="ghost"
+      className={`font-medium border-none hover:bg-black  px-2 h-10 ${className}`}
+      {...props}>
+      {props.children}
+    </Button>
+  );
+}
+
+export default function ObjectActions({objectId, active}: Props) {
+  const [isRemovingObject, setIsRemovingObject] = useState<boolean>(false);
+  const [activeTrackId, setActiveTrackletId] = useAtom(
+    activeTrackletObjectIdAtom,
+  );
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const isPlaying = useAtom(isPlayingAtom);
+
+  const video = useVideo();
+  const reportError = useReportError();
+
+  async function handleRemoveObject(
+    event: React.MouseEvent<HTMLButtonElement>,
+  ) {
+    try {
+      event.stopPropagation();
+      setIsRemovingObject(true);
+      if (isStreaming) {
+        await video?.abortStreamMasks();
+      }
+      if (isPlaying) {
+        video?.pause();
+      }
+      await video?.deleteTracklet(objectId);
+    } catch (error) {
+      reportError(error);
+    } finally {
+      setIsRemovingObject(false);
+      if (activeTrackId === objectId) {
+        setActiveTrackletId(null);
+      }
+    }
+  }
+
+  return (
+    <div>
+      {active && (
+        <div className="text-sm mt-1 leading-snug text-gray-400 hidden md:block ml-2 md:mb-4">
+          Select <AddFilled size={14} className="inline" /> to add areas to the
+          object and <SubtractFilled size={14} className="inline" /> to remove
+          areas from the object in the video. Click on an existing point to
+          delete it.
+        </div>
+      )}
+
+      <div className="flex justify-between items-center md:mt-2 mt-0">
+        {active ? (
+          <PointsToggle />
+        ) : (
+          <>
+            <CustomButton startIcon={<Select_02 size={24} />}>
+              Edit selection
+            </CustomButton>
+            <CustomButton
+              loading={isRemovingObject}
+              onClick={handleRemoveObject}
+              startIcon={!isRemovingObject && <TrashCan size={24} />}>
+              <span className="hidden md:inline">Clear</span>
+            </CustomButton>
+          </>
+        )}
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectPlaceholder.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+
+type Props = {
+  showPlus?: boolean;
+  onClick?: () => void;
+};
+
+export default function ObjectPlaceholder({showPlus = true, onClick}: Props) {
+  return (
+    <div
+      className={`relative ${BLUE_PINK_FILL_BR} h-12 w-12 md:h-20 md:w-20 shrink-0 rounded-lg`}
+      onClick={onClick}>
+      {showPlus && (
+        <div className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2">
+          <svg
+            xmlns="http://www.w3.org/2000/svg"
+            width="16"
+            height="16"
+            viewBox="0 0 16 16"
+            fill="none">
+            <path
+              d="M16 7H9V0H7V7H0V9H7V16H9V9H16V7Z"
+              fill="#667788"
+              fillOpacity={1}
+            />
+          </svg>
+        </div>
+      )}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectThumbnail.tsx

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type Props = {
+  thumbnail: string | null;
+  color: string;
+  onClick?: () => void;
+};
+
+export default function ObjectThumbnail({thumbnail, color, onClick}: Props) {
+  return (
+    <div
+      className="relative h-12 w-12 md:w-20 md:h-20 shrink-0 p-2 rounded-lg bg-contain bg-no-repeat bg-center"
+      style={{
+        backgroundColor: color,
+      }}
+      onClick={onClick}>
+      <div
+        className="w-full h-full bg-contain bg-no-repeat bg-center"
+        style={{
+          backgroundImage: thumbnail == null ? 'none' : `url(${thumbnail})`,
+        }}></div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectUtils.ts

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BaseTracklet} from '@/common/tracker/Tracker';
+
+export function getObjectLabel(tracklet: BaseTracklet) {
+  return `Object ${tracklet.id + 1}`;
+}

demo/frontend/src/common/components/annotations/ObjectsToolbar.tsx

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import AddObjectButton from '@/common/components/annotations/AddObjectButton';
+import FirstClickView from '@/common/components/annotations/FirstClickView';
+import LimitNotice from '@/common/components/annotations/LimitNotice';
+import ObjectsToolbarBottomActions from '@/common/components/annotations/ObjectsToolbarBottomActions';
+import ObjectsToolbarHeader from '@/common/components/annotations/ObjectsToolbarHeader';
+import {getObjectLabel} from '@/common/components/annotations/ObjectUtils';
+import ToolbarObject from '@/common/components/annotations/ToolbarObject';
+import {
+  activeTrackletObjectAtom,
+  activeTrackletObjectIdAtom,
+  isAddObjectEnabledAtom,
+  isFirstClickMadeAtom,
+  isTrackletObjectLimitReachedAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function ObjectsToolbar({onTabChange}: Props) {
+  const tracklets = useAtomValue(trackletObjectsAtom);
+  const activeTracklet = useAtomValue(activeTrackletObjectAtom);
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+  const isFirstClickMade = useAtomValue(isFirstClickMadeAtom);
+  const isObjectLimitReached = useAtomValue(isTrackletObjectLimitReachedAtom);
+  const isAddObjectEnabled = useAtomValue(isAddObjectEnabledAtom);
+
+  if (!isFirstClickMade) {
+    return <FirstClickView />;
+  }
+
+  return (
+    <div className="flex flex-col h-full">
+      <ObjectsToolbarHeader />
+      <div className="grow w-full overflow-y-auto">
+        {tracklets.map(tracklet => {
+          return (
+            <ToolbarObject
+              key={tracklet.id}
+              label={getObjectLabel(tracklet)}
+              tracklet={tracklet}
+              isActive={activeTracklet?.id === tracklet.id}
+              onClick={() => {
+                setActiveTrackletId(tracklet.id);
+              }}
+            />
+          );
+        })}
+        {isAddObjectEnabled && <AddObjectButton />}
+        {isObjectLimitReached && <LimitNotice />}
+      </div>
+      <ObjectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectsToolbarBottomActions.tsx

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ClearAllPointsInVideoButton from '@/common/components/annotations/ClearAllPointsInVideoButton';
+import CloseSessionButton from '@/common/components/annotations/CloseSessionButton';
+import TrackAndPlayButton from '@/common/components/button/TrackAndPlayButton';
+import ToolbarBottomActionsWrapper from '@/common/components/toolbar/ToolbarBottomActionsWrapper';
+import {
+  EFFECT_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function ObjectsToolbarBottomActions({onTabChange}: Props) {
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  const isTrackingEnabled =
+    streamingState !== 'none' && streamingState !== 'full';
+
+  function handleSwitchToEffectsTab() {
+    onTabChange(EFFECT_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <ClearAllPointsInVideoButton
+        onRestart={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+      {isTrackingEnabled && <TrackAndPlayButton />}
+      {streamingState === 'full' && (
+        <CloseSessionButton onSessionClose={handleSwitchToEffectsTab} />
+      )}
+    </ToolbarBottomActionsWrapper>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectsToolbarHeader.tsx

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function ObjectsToolbarHeader() {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  return (
+    <ToolbarHeaderWrapper
+      title={
+        streamingState === 'full'
+          ? 'Review tracked objects'
+          : isStreaming
+            ? 'Tracking objects'
+            : 'Select objects'
+      }
+      description={
+        streamingState === 'full'
+          ? 'Review your selected objects across the video, and continue to edit if needed. Once everything looks good, press “Next” to continue.'
+          : isStreaming
+            ? 'Watch the video closely for any places where your objects aren’t tracked correctly. You can also stop tracking to make additional edits.'
+            : 'Adjust the selection of your object, or add additional objects. Press “Track objects” to track your objects throughout the video.'
+      }
+      className="mb-8"
+    />
+  );
+}

demo/frontend/src/common/components/annotations/PointsToggle.tsx

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {labelTypeAtom} from '@/demo/atoms';
+import {AddFilled, SubtractFilled} from '@carbon/icons-react';
+import {useAtom} from 'jotai';
+
+export default function PointsToggle() {
+  const [labelType, setLabelType] = useAtom(labelTypeAtom);
+  const isPositive = labelType === 'positive';
+
+  const buttonStyle = (selected: boolean) =>
+    `btn-md bg-graydark-800 !text-white md:px-2 lg:px-4 py-0.5 ${selected ? `border border-white hover:bg-graydark-800` : `border-graydark-700 hover:bg-graydark-700`}`;
+
+  return (
+    <div className="flex items-center w-full md:ml-2">
+      <div className="join group grow gap-[1px]">
+        <button
+          className={`w-1/2  btn join-item text-white ${buttonStyle(isPositive)}`}
+          onClick={() => setLabelType('positive')}>
+          <AddFilled size={24} className="text-blue-500" /> Add
+        </button>
+        <button
+          className={`w-1/2 btn join-item text-red-700 ${buttonStyle(!isPositive)}`}
+          onClick={() => setLabelType('negative')}>
+          <SubtractFilled size={24} className="text-red-400" />
+          Remove
+        </button>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/PrimaryCTAButton.tsx

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+  disabled?: boolean;
+  endIcon?: ReactNode;
+} & React.DOMAttributes<HTMLButtonElement>;
+
+export default function PrimaryCTAButton({
+  children,
+  disabled,
+  endIcon,
+  ...props
+}: Props) {
+  return (
+    <GradientBorder disabled={disabled}>
+      <button
+        className={`btn ${disabled && 'btn-disabled'} !rounded-full !bg-black !text-white !border-none`}
+        {...props}>
+        {children}
+        {endIcon != null && endIcon}
+      </button>
+    </GradientBorder>
+  );
+}

demo/frontend/src/common/components/annotations/ToolbarObject.tsx

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ObjectActions from '@/common/components/annotations/ObjectActions';
+import ObjectPlaceholder from '@/common/components/annotations/ObjectPlaceholder';
+import ObjectThumbnail from '@/common/components/annotations/ObjectThumbnail';
+import ToolbarObjectContainer from '@/common/components/annotations/ToolbarObjectContainer';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import emptyFunction from '@/common/utils/emptyFunction';
+import {activeTrackletObjectIdAtom} from '@/demo/atoms';
+import {useSetAtom} from 'jotai';
+
+type Props = {
+  label: string;
+  tracklet: BaseTracklet;
+  isActive: boolean;
+  isMobile?: boolean;
+  onClick?: () => void;
+  onThumbnailClick?: () => void;
+};
+
+export default function ToolbarObject({
+  label,
+  tracklet,
+  isActive,
+  isMobile = false,
+  onClick,
+  onThumbnailClick = emptyFunction,
+}: Props) {
+  const video = useVideo();
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+
+  async function handleCancelNewObject() {
+    try {
+      await video?.deleteTracklet(tracklet.id);
+    } catch (error) {
+      reportError(error);
+    } finally {
+      setActiveTrackletId(null);
+    }
+  }
+
+  if (!tracklet.isInitialized) {
+    return (
+      <ToolbarObjectContainer
+        alignItems="center"
+        isActive={isActive}
+        title="New object"
+        subtitle="No object is currently selected. Click an object in the video."
+        thumbnail={<ObjectPlaceholder showPlus={false} />}
+        isMobile={isMobile}
+        onClick={onClick}
+        onCancel={handleCancelNewObject}
+      />
+    );
+  }
+
+  return (
+    <ToolbarObjectContainer
+      isActive={isActive}
+      onClick={onClick}
+      title={label}
+      subtitle=""
+      thumbnail={
+        <ObjectThumbnail
+          thumbnail={tracklet.thumbnail}
+          color={tracklet.color}
+          onClick={onThumbnailClick}
+        />
+      }
+      isMobile={isMobile}>
+      <ObjectActions objectId={tracklet.id} active={isActive} />
+    </ToolbarObjectContainer>
+  );
+}

demo/frontend/src/common/components/annotations/ToolbarObjectContainer.tsx

@@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren, ReactNode} from 'react';
+
+const sharedStyles = stylex.create({
+  container: {
+    display: 'flex',
+    overflow: 'hidden',
+    cursor: 'pointer',
+    flexShrink: 0,
+    borderTop: 'none',
+    backgroundColor: {
+      '@media screen and (max-width: 768px)': '#000',
+    },
+    paddingHorizontal: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    paddingBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 10,
+    },
+  },
+  activeContainer: {
+    background: '#000',
+    borderRadius: 16,
+    marginHorizontal: 16,
+    padding: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    marginBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 0,
+    },
+  },
+  itemsCenter: {
+    alignItems: 'center',
+  },
+  rightColumn: {
+    marginStart: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': 0,
+    },
+    flexGrow: 1,
+    alignItems: 'center',
+  },
+});
+
+type ToolbarObjectContainerProps = PropsWithChildren<{
+  alignItems?: 'top' | 'center';
+  isActive: boolean;
+  title: string;
+  subtitle: string;
+  thumbnail: ReactNode;
+  isMobile: boolean;
+  onCancel?: () => void;
+  onClick?: () => void;
+}>;
+
+export default function ToolbarObjectContainer({
+  alignItems = 'top',
+  children,
+  isActive,
+  title,
+  subtitle,
+  thumbnail,
+  isMobile,
+  onClick,
+  onCancel,
+}: ToolbarObjectContainerProps) {
+  if (isMobile) {
+    return (
+      <div
+        onClick={onClick}
+        {...stylex.props(sharedStyles.container, sharedStyles.itemsCenter)}>
+        <div {...stylex.props(sharedStyles.rightColumn)}>{children}</div>
+      </div>
+    );
+  }
+
+  return (
+    <div
+      onClick={onClick}
+      {...stylex.props(
+        sharedStyles.container,
+        isActive && sharedStyles.activeContainer,
+        alignItems === 'center' && sharedStyles.itemsCenter,
+      )}>
+      {thumbnail}
+      <div {...stylex.props(sharedStyles.rightColumn)}>
+        <div className="text-md font-semibold ml-2">{title}</div>
+        {subtitle.length > 0 && (
+          <div className="text-sm text-gray-400 leading-5 mt-2 ml-2">
+            {subtitle}
+          </div>
+        )}
+        {children}
+      </div>
+      {onCancel != null && (
+        <div className="items-start self-stretch" onClick={onCancel}>
+          <Close size={32} />
+        </div>
+      )}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/TrackletSwimlane.tsx

@@ -0,0 +1,173 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useSelectedFrameHelper from '@/common/components/video/filmstrip/useSelectedFrameHelper';
+import {BaseTracklet, DatalessMask} from '@/common/tracker/Tracker';
+import {spacing, w} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[4],
+    width: '100%',
+  },
+  trackletNameContainer: {
+    width: w[12],
+    textAlign: 'center',
+    fontSize: '10px',
+    color: 'white',
+  },
+  swimlaneContainer: {
+    flexGrow: 1,
+    position: 'relative',
+    display: 'flex',
+    height: 12,
+    marginVertical: '0.25rem' /* 4px */,
+    '@media screen and (max-width: 768px)': {
+      marginVertical: 0,
+    },
+  },
+  swimlane: {
+    position: 'absolute',
+    left: 0,
+    top: '50%',
+    width: '100%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+    opacity: 0.4,
+  },
+  segment: {
+    position: 'absolute',
+    top: '50%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+  },
+  segmentationPoint: {
+    position: 'absolute',
+    top: '50%',
+    transform: 'translate3d(0, -50%, 0)',
+    borderRadius: '50%',
+    cursor: 'pointer',
+    width: 12,
+    height: 12,
+    '@media screen and (max-width: 768px)': {
+      width: 8,
+      height: 8,
+    },
+  },
+});
+
+type SwimlineSegment = {
+  left: number;
+  width: number;
+};
+
+type Props = {
+  tracklet: BaseTracklet;
+  onSelectFrame: (tracklet: BaseTracklet, index: number) => void;
+};
+
+function getSwimlaneSegments(masks: DatalessMask[]): SwimlineSegment[] {
+  if (masks.length === 0) {
+    return [];
+  }
+
+  const swimlineSegments: SwimlineSegment[] = [];
+  let left = -1;
+
+  for (let frameIndex = 0; frameIndex < masks.length; ++frameIndex) {
+    const isEmpty = masks?.[frameIndex]?.isEmpty ?? true;
+    if (left === -1 && !isEmpty) {
+      left = frameIndex;
+    } else if (left !== -1 && (isEmpty || frameIndex == masks.length - 1)) {
+      swimlineSegments.push({
+        left,
+        width: frameIndex - left + 1,
+      });
+      left = -1;
+    }
+  }
+
+  return swimlineSegments;
+}
+
+export default function TrackletSwimlane({tracklet, onSelectFrame}: Props) {
+  const selection = useSelectedFrameHelper();
+
+  const segments = useMemo(() => {
+    return getSwimlaneSegments(tracklet.masks);
+  }, [tracklet.masks]);
+
+  const framesWithPoints = tracklet.points.reduce<number[]>(
+    (frames, pts, frameIndex) => {
+      if (pts != null && pts.length > 0) {
+        frames.push(frameIndex);
+      }
+      return frames;
+    },
+    [],
+  );
+
+  if (selection === null) {
+    return;
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.trackletNameContainer)}>
+        Object {tracklet.id + 1}
+      </div>
+      <div {...stylex.props(styles.swimlaneContainer)}>
+        <div
+          {...stylex.props(styles.swimlane)}
+          style={{
+            backgroundColor: tracklet.color,
+          }}
+        />
+        {segments.map(segment => {
+          return (
+            <div
+              key={segment.left}
+              {...stylex.props(styles.segment)}
+              style={{
+                backgroundColor: tracklet.color,
+                left: selection.toPosition(segment.left),
+                width: selection.toPosition(segment.width),
+              }}
+            />
+          );
+        })}
+        {framesWithPoints.map(index => {
+          return (
+            <div
+              key={`frame${index}`}
+              onClick={() => {
+                onSelectFrame?.(tracklet, index);
+              }}
+              {...stylex.props(styles.segmentationPoint)}
+              style={{
+                left: Math.floor(selection.toPosition(index) - 4),
+                backgroundColor: tracklet.color,
+              }}
+            />
+          );
+        })}
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/TrackletsAnnotation.tsx

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletSwimlane from '@/common/components/annotations/TrackletSwimlane';
+import useTracklets from '@/common/components/annotations/useTracklets';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import {m, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    marginTop: m[3],
+    height: 75,
+    paddingHorizontal: spacing[4],
+    '@media screen and (max-width: 768px)': {
+      height: 25,
+    },
+  },
+});
+
+export default function TrackletsAnnotation() {
+  const video = useVideo();
+  const tracklets = useTracklets();
+
+  function handleSelectFrame(_tracklet: BaseTracklet, index: number) {
+    if (video !== null) {
+      video.frame = index;
+    }
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      {tracklets.map(tracklet => (
+        <TrackletSwimlane
+          key={tracklet.id}
+          tracklet={tracklet}
+          onSelectFrame={handleSelectFrame}
+        />
+      ))}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/useTracklets.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {trackletObjectsAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function useTracklets() {
+  return useAtomValue(trackletObjectsAtom);
+}

demo/frontend/src/common/components/button/GradientBorder.tsx

@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import stylex from '@stylexjs/stylex';
+
+import {gradients} from '@/theme/tokens.stylex';
+
+enum GradientTypes {
+  fullGradient = 'fullGradient',
+  bluePinkGradient = 'bluePinkGradient',
+}
+
+type Props = {
+  gradientType?: GradientTypes;
+  disabled?: boolean;
+  rounded?: boolean;
+  className?: string;
+} & React.DOMAttributes<HTMLDivElement>;
+
+const styles = stylex.create({
+  animationHover: {
+    ':hover': {
+      backgroundPosition: '300% 100%',
+    },
+  },
+
+  fullGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+    backgroundSize: '100% 400%',
+    transition: 'background 0.35s ease-in-out',
+  },
+
+  bluePinkGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+  },
+});
+
+export default function GradientBorder({
+  gradientType = GradientTypes.fullGradient,
+  disabled,
+  rounded = true,
+  className = '',
+  children,
+}: Props) {
+  const gradient = (name: GradientTypes) => {
+    if (name === GradientTypes.fullGradient) {
+      return styles.fullGradient;
+    } else if (name === GradientTypes.bluePinkGradient) {
+      return styles.bluePinkGradient;
+    }
+  };
+
+  return (
+    <div
+      className={`${stylex(gradient(gradientType), !disabled && styles.animationHover)} ${disabled && 'opacity-30'} ${rounded && 'rounded-full'} ${className}`}>
+      {children}
+    </div>
+  );
+}

demo/frontend/src/common/components/button/PlaybackButton.tsx

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import Tooltip from '@/common/components/Tooltip';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {isPlayingAtom, streamingStateAtom, toolbarTabIndex} from '@/demo/atoms';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import {useAtomValue} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function PlaybackButton() {
+  const tabIndex = useAtomValue(toolbarTabIndex);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const video = useVideo();
+
+  const isDisabled =
+    tabIndex === OBJECT_TOOLBAR_INDEX &&
+    streamingState !== 'none' &&
+    streamingState !== 'full';
+
+  const handlePlay = useCallback(() => {
+    video?.play();
+  }, [video]);
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  const handleClick = useCallback(() => {
+    if (isDisabled) {
+      return;
+    }
+    if (isPlaying) {
+      handlePause();
+    } else {
+      handlePlay();
+    }
+  }, [isDisabled, isPlaying, handlePlay, handlePause]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleClick,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleClick]);
+
+  return (
+    <Tooltip message={`${isPlaying ? 'Pause' : 'Play'} (k)`}>
+      <button
+        disabled={isDisabled}
+        className={`group !rounded-full !w-10 !h-10 flex items-center justify-center ${getButtonStyles(isDisabled)}`}
+        onClick={handleClick}>
+        {isPlaying ? (
+          <PauseFilled size={18} />
+        ) : (
+          <PlayFilledAlt
+            size={18}
+            className={!isDisabled ? 'group-hover:text-green-500' : ''}
+          />
+        )}
+      </button>
+    </Tooltip>
+  );
+}
+
+function getButtonStyles(isDisabled: boolean): string {
+  if (isDisabled) {
+    return '!bg-gray-600 !text-graydark-700';
+  }
+  return `!text-black bg-white`;
+}

demo/frontend/src/common/components/button/PrimaryCTAButton.tsx

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+  disabled?: boolean;
+  endIcon?: ReactNode;
+} & React.DOMAttributes<HTMLButtonElement>;
+
+export default function PrimaryCTAButton({
+  children,
+  disabled,
+  endIcon,
+  ...props
+}: Props) {
+  return (
+    <GradientBorder disabled={disabled}>
+      <button
+        className={`btn ${disabled && 'btn-disabled'} !rounded-full !bg-black !text-white !border-none`}
+        {...props}>
+        {children}
+        {endIcon != null && endIcon}
+      </button>
+    </GradientBorder>
+  );
+}

demo/frontend/src/common/components/button/ResponsiveButton.tsx

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useScreenSize from '@/common/screen/useScreenSize';
+import type {ReactNode} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = ButtonProps & {startIcon: ReactNode};
+
+export default function ResponsiveButton(props: Props) {
+  const {isMobile} = useScreenSize();
+
+  return <Button {...props}>{!isMobile && props.children}</Button>;
+}

demo/frontend/src/common/components/button/TrackAndPlayButton.tsx

@@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useFunctionThrottle from '@/common/components/useFunctionThrottle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  areTrackletObjectsInitializedAtom,
+  isStreamingAtom,
+  sessionAtom,
+  streamingStateAtom,
+} from '@/demo/atoms';
+import {ChevronRight} from '@carbon/icons-react';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function TrackAndPlayButton() {
+  const video = useVideo();
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const areObjectsInitialized = useAtomValue(areTrackletObjectsInitializedAtom);
+  const setSession = useSetAtom(sessionAtom);
+  const {enqueueMessage} = useMessagesSnackbar();
+  const {isThrottled, maxThrottles, throttle} = useFunctionThrottle(250, 4);
+
+  const isTrackAndPlayDisabled =
+    streamingState === 'aborting' || streamingState === 'requesting';
+
+  useEffect(() => {
+    function onStreamingStarted() {
+      setIsStreaming(true);
+    }
+    video?.addEventListener('streamingStarted', onStreamingStarted);
+
+    function onStreamingCompleted() {
+      enqueueMessage('trackAndPlayComplete');
+      setIsStreaming(false);
+    }
+    video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+    return () => {
+      video?.removeEventListener('streamingStarted', onStreamingStarted);
+      video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+    };
+  }, [video, setIsStreaming, enqueueMessage]);
+
+  const handleTrackAndPlay = useCallback(() => {
+    if (isTrackAndPlayDisabled) {
+      return;
+    }
+    if (maxThrottles && isThrottled) {
+      enqueueMessage('trackAndPlayThrottlingWarning');
+    }
+
+    // Throttling is only applied while streaming because we should
+    // only throttle after a user has aborted inference. This way,
+    // a user can still quickly abort a stream if they notice the
+    // inferred mask is misaligned.
+    throttle(
+      () => {
+        if (!isStreaming) {
+          enqueueMessage('trackAndPlayClick');
+          video?.streamMasks();
+          setSession(previousSession =>
+            previousSession == null
+              ? previousSession
+              : {...previousSession, ranPropagation: true},
+          );
+        } else {
+          video?.abortStreamMasks();
+        }
+      },
+      {enableThrottling: isStreaming},
+    );
+  }, [
+    isTrackAndPlayDisabled,
+    isThrottled,
+    isStreaming,
+    maxThrottles,
+    video,
+    setSession,
+    enqueueMessage,
+    throttle,
+  ]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleTrackAndPlay,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleTrackAndPlay]);
+
+  return (
+    <PrimaryCTAButton
+      disabled={isThrottled || !areObjectsInitialized}
+      onClick={handleTrackAndPlay}
+      endIcon={isStreaming ? undefined : <ChevronRight size={20} />}>
+      {isStreaming ? 'Cancel Tracking' : 'Track objects'}
+    </PrimaryCTAButton>
+  );
+}

demo/frontend/src/common/components/code/InitializeLocalMonaco.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {loader} from '@monaco-editor/react';
+
+import Logger from '@/common/logger/Logger';
+import * as monaco from 'monaco-editor';
+import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker';
+import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker';
+
+self.MonacoEnvironment = {
+  getWorker(_, label) {
+    if (label === 'typescript' || label === 'javascript') {
+      return new tsWorker();
+    }
+    return new editorWorker();
+  },
+};
+
+loader.config({monaco});
+
+loader.init().then(monaco => {
+  Logger.debug('initialized monaco', monaco);
+});

demo/frontend/src/common/components/effects/BackgroundEffects.tsx

@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {backgroundEffects} from '@/common/components/effects/EffectsUtils';
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {activeBackgroundEffectAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function BackgroundEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeBackgroundEffectAtom);
+
+  return (
+    <ToolbarSection title="Background" borderBottom={false}>
+      {backgroundEffects.map(backgroundEffect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={backgroundEffect.title}
+            icon={backgroundEffect.Icon}
+            title={backgroundEffect.title}
+            isActive={activeEffect.name === backgroundEffect.effectName}
+            badge={
+              activeEffect.name === backgroundEffect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              if (activeEffect.name === backgroundEffect.effectName) {
+                setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND, {
+                  variant:
+                    (activeEffect.variant + 1) % activeEffect.numVariants,
+                });
+              } else {
+                setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND);
+              }
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}

demo/frontend/src/common/components/effects/EffectVariantBadge.tsx

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {right, top} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  variantBadge: {
+    position: 'absolute',
+    top: top[1],
+    right: right[1],
+    backgroundColor: '#280578',
+    color: '#D2D2FF',
+    fontVariantNumeric: 'tabular-nums',
+    paddingHorizontal: 4,
+    paddingVertical: 1,
+    fontSize: 9,
+    borderRadius: 6,
+    fontWeight: 'bold',
+  },
+});
+
+type Props = {
+  label: string;
+};
+
+export default function VariantBadge({label}: Props) {
+  return <div {...stylex.props(styles.variantBadge)}>{label}</div>;
+}

demo/frontend/src/common/components/effects/EffectsCarousel.tsx

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {CarouselContainerShadow} from '@/common/components/effects/EffectsCarouselShadow';
+import {DemoEffect} from '@/common/components/effects/EffectsUtils';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import type {EffectIndex} from '@/common/components/video/effects/Effects';
+import {Effects} from '@/common/components/video/effects/Effects';
+import {color, fontSize, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+type Props = {
+  label: string;
+  effects: DemoEffect[];
+  activeEffect: keyof Effects;
+  index: EffectIndex;
+};
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing[2],
+    width: '100%',
+  },
+  label: {
+    fontSize: fontSize['xs'],
+    color: '#A6ACB2',
+    textAlign: 'center',
+  },
+  carouselContainer: {
+    position: 'relative',
+    borderRadius: '8px',
+    overflow: 'hidden',
+    width: '100%',
+    height: '120px',
+    backgroundColor: color['gray-700'],
+  },
+});
+
+export default function EffectsCarousel({
+  label,
+  effects,
+  activeEffect,
+  index: effectIndex,
+}: Props) {
+  const setEffect = useVideoEffect();
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.label)}>{label}</div>
+      <div {...stylex.props(styles.carouselContainer)}>
+        <CarouselContainerShadow isTop={true} />
+        <div className="carousel carousel-vertical w-full h-full text-white">
+          <div className={`carousel-item h-6`} />
+          {effects.map(({effectName, Icon, title}, index) => {
+            const isActive = activeEffect === effectName;
+            return (
+              <div
+                key={index}
+                className={`carousel-item flex items-center h-6 gap-2 px-4`}
+                onClick={() => setEffect(effectName, effectIndex)}>
+                <Icon
+                  color={isActive ? '#FB73A5' : undefined}
+                  size={18}
+                  fontWeight={10}
+                />
+                <div
+                  className={`text-sm ${isActive ? 'text-[#FB73A5] font-bold' : 'font-medium'}`}>
+                  {title}
+                </div>
+              </div>
+            );
+          })}
+          <div className={`carousel-item h-6`} />
+        </div>
+        <CarouselContainerShadow isTop={false} />
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/effects/EffectsCarouselShadow.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    width: '100%',
+    height: spacing[8],
+    pointerEvents: 'none',
+  },
+});
+
+type CarouselContainerShadowProps = {
+  isTop: boolean;
+};
+
+const edgeColor = 'rgba(55, 62, 65, 1)';
+const transitionColor = 'rgba(55, 62, 65, 0.2)';
+
+export function CarouselContainerShadow({isTop}: CarouselContainerShadowProps) {
+  return (
+    <div
+      {...stylex.props(styles.container)}
+      style={{
+        background: `linear-gradient(${isTop ? `${edgeColor}, ${transitionColor}` : `${transitionColor}, ${edgeColor}`})`,
+        top: isTop ? 0 : undefined,
+        bottom: isTop ? undefined : 0,
+      }}
+    />
+  );
+}

demo/frontend/src/common/components/effects/EffectsToolbar.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BackgroundEffects from '@/common/components/effects/BackgroundEffects';
+import EffectsToolbarBottomActions from '@/common/components/effects/EffectsToolbarBottomActions';
+import EffectsToolbarHeader from '@/common/components/effects/EffectsToolbarHeader';
+import HighlightEffects from '@/common/components/effects/HighlightEffects';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {useEffect, useRef} from 'react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function EffectsToolbar({onTabChange}: Props) {
+  const isEffectsMessageShown = useRef(false);
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useEffect(() => {
+    if (!isEffectsMessageShown.current) {
+      isEffectsMessageShown.current = true;
+      enqueueMessage('effectsMessage');
+    }
+  }, [enqueueMessage]);
+
+  return (
+    <div className="flex flex-col h-full">
+      <EffectsToolbarHeader />
+      <div className="grow overflow-y-auto">
+        <HighlightEffects />
+        <BackgroundEffects />
+      </div>
+      <EffectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

demo/frontend/src/common/components/effects/EffectsToolbarBottomActions.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import RestartSessionButton from '@/common/components/session/RestartSessionButton';
+import ToolbarBottomActionsWrapper from '@/common/components/toolbar/ToolbarBottomActionsWrapper';
+import {
+  MORE_OPTIONS_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {ChevronRight} from '@carbon/icons-react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function EffectsToolbarBottomActions({onTabChange}: Props) {
+  function handleSwitchToMoreOptionsTab() {
+    onTabChange(MORE_OPTIONS_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <RestartSessionButton
+        onRestartSession={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+      <PrimaryCTAButton
+        onClick={handleSwitchToMoreOptionsTab}
+        endIcon={<ChevronRight />}>
+        Next
+      </PrimaryCTAButton>
+    </ToolbarBottomActionsWrapper>
+  );
+}

demo/frontend/src/common/components/effects/EffectsToolbarHeader.tsx

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {
+  EffectIndex,
+  effectPresets,
+} from '@/common/components/video/effects/Effects';
+import {BLUE_PINK_FILL} from '@/theme/gradientStyle';
+import {MagicWandFilled} from '@carbon/icons-react';
+import {useCallback, useRef} from 'react';
+import {Button} from 'react-daisyui';
+
+export default function EffectsToolbarHeader() {
+  const preset = useRef(0);
+  const setEffect = useVideoEffect();
+
+  const handleTogglePreset = useCallback(() => {
+    preset.current++;
+    const [background, highlight] =
+      effectPresets[preset.current % effectPresets.length];
+    setEffect(background.name, EffectIndex.BACKGROUND, {
+      variant: background.variant,
+    });
+    setEffect(highlight.name, EffectIndex.HIGHLIGHT, {
+      variant: highlight.variant,
+    });
+  }, [setEffect]);
+
+  return (
+    <ToolbarHeaderWrapper
+      title="Add effects"
+      description="Apply visual effects to your selected objects and the background. Keeping clicking the same effect for different variations."
+      bottomSection={
+        <div className="flex mt-1">
+          <Button
+            color="ghost"
+            size="md"
+            className={`font-medium bg-black !rounded-full hover:!bg-gradient-to-br ${BLUE_PINK_FILL} border-none`}
+            endIcon={<MagicWandFilled size={20} className="text-white " />}
+            onClick={handleTogglePreset}>
+            Surprise Me
+          </Button>
+        </div>
+      }
+      className="pb-4"
+    />
+  );
+}

demo/frontend/src/common/components/effects/EffectsUtils.ts

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Effects} from '@/common/components/video/effects/Effects';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {
+  AppleDash,
+  Asterisk,
+  Barcode,
+  CenterCircle,
+  ColorPalette,
+  ColorSwitch,
+  Development,
+  Erase,
+  FaceWink,
+  Humidity,
+  Image,
+  Overlay,
+  TextFont,
+} from '@carbon/icons-react';
+
+export type DemoEffect = {
+  title: string;
+  Icon: CarbonIconType;
+  effectName: keyof Effects;
+};
+
+export const backgroundEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Original'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseBackground'},
+  {
+    title: 'Gradient',
+    Icon: ColorPalette,
+    effectName: 'Gradient',
+  },
+  {
+    title: 'Pixelate',
+    Icon: Development,
+    effectName: 'Pixelate',
+  },
+  {title: 'Desaturate', Icon: ColorSwitch, effectName: 'Desaturate'},
+  {title: 'Text', Icon: TextFont, effectName: 'BackgroundText'},
+  {title: 'Blur', Icon: Humidity, effectName: 'BackgroundBlur'},
+  {title: 'Outline', Icon: AppleDash, effectName: 'Sobel'},
+];
+
+export const highlightEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Cutout'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseForeground'},
+  {title: 'Gradient', Icon: ColorPalette, effectName: 'VibrantMask'},
+  {title: 'Pixelate', Icon: Development, effectName: 'PixelateMask'},
+  {
+    title: 'Overlay',
+    Icon: Overlay,
+    effectName: 'Overlay',
+  },
+  {title: 'Emoji', Icon: FaceWink, effectName: 'Replace'},
+  {title: 'Burst', Icon: Asterisk, effectName: 'Burst'},
+  {title: 'Spotlight', Icon: CenterCircle, effectName: 'Scope'},
+];
+
+export const moreEffects: DemoEffect[] = [
+  {title: 'Noisy', Icon: Barcode, effectName: 'NoisyMask'},
+];

demo/frontend/src/common/components/effects/HighlightEffects.tsx

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {
+  activeHighlightEffectAtom,
+  activeHighlightEffectGroupAtom,
+} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function HighlightEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeHighlightEffectAtom);
+  const activeEffectsGroup = useAtomValue(activeHighlightEffectGroupAtom);
+
+  return (
+    <ToolbarSection title="Selected Objects" borderBottom={true}>
+      {activeEffectsGroup.map(highlightEffect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={highlightEffect.title}
+            icon={highlightEffect.Icon}
+            title={highlightEffect.title}
+            isActive={activeEffect.name === highlightEffect.effectName}
+            badge={
+              activeEffect.name === highlightEffect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              if (activeEffect.name === highlightEffect.effectName) {
+                setEffect(highlightEffect.effectName, EffectIndex.HIGHLIGHT, {
+                  variant:
+                    (activeEffect.variant + 1) % activeEffect.numVariants,
+                });
+              } else {
+                setEffect(highlightEffect.effectName, EffectIndex.HIGHLIGHT);
+              }
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}

demo/frontend/src/common/components/effects/MobileEffectsToolbar.tsx

@@ -0,0 +1,115 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import EffectsCarousel from '@/common/components/effects/EffectsCarousel';
+import {backgroundEffects} from '@/common/components/effects/EffectsUtils';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {
+  EffectIndex,
+  effectPresets,
+} from '@/common/components/video/effects/Effects';
+import {ListBoxes, MagicWand, MagicWandFilled} from '@carbon/icons-react';
+import {useCallback, useRef, useState} from 'react';
+import {Button} from 'react-daisyui';
+
+import EffectsToolbarBottomActions from '@/common/components/effects/EffectsToolbarBottomActions';
+import ToolbarProgressChip from '@/common/components/toolbar/ToolbarProgressChip';
+import {
+  activeBackgroundEffectAtom,
+  activeHighlightEffectAtom,
+  activeHighlightEffectGroupAtom,
+} from '@/demo/atoms';
+import {BLUE_PINK_FILL} from '@/theme/gradientStyle';
+import {useAtomValue} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileEffectsToolbar({onTabChange}: Props) {
+  const preset = useRef(0);
+  const setEffect = useVideoEffect();
+  const [showEffectsCarousels, setShowEffectsCarousels] = useState<boolean>();
+
+  const activeBackground = useAtomValue(activeBackgroundEffectAtom);
+  const activeHighlight = useAtomValue(activeHighlightEffectAtom);
+  const activeHighlightEffectsGroup = useAtomValue(
+    activeHighlightEffectGroupAtom,
+  );
+
+  const handleTogglePreset = useCallback(() => {
+    preset.current++;
+    const [background, highlight] =
+      effectPresets[preset.current % effectPresets.length];
+    setEffect(background.name, EffectIndex.BACKGROUND, {
+      variant: background.variant,
+    });
+    setEffect(highlight.name, EffectIndex.HIGHLIGHT, {
+      variant: highlight.variant,
+    });
+  }, [setEffect]);
+
+  return (
+    <div className="w-full">
+      {showEffectsCarousels ? (
+        <div className="flex gap-2 px-2 py-4 items-center p-6">
+          <Button
+            color="ghost"
+            className="mt-6 !px-2 !text-[#FB73A5]"
+            startIcon={<MagicWand size={20} />}
+            onClick={handleTogglePreset}
+          />
+          <EffectsCarousel
+            label="Highlights"
+            effects={activeHighlightEffectsGroup}
+            activeEffect={activeHighlight.name}
+            index={1}
+          />
+          <EffectsCarousel
+            label="Background"
+            effects={backgroundEffects}
+            activeEffect={activeBackground.name}
+            index={0}
+          />
+        </div>
+      ) : (
+        <div className="flex flex-col gap-6 p-6">
+          <div className="text-sm text-white">
+            <ToolbarProgressChip />
+            Apply visual effects to your selected objects and the background.
+          </div>
+          <div className="grid grid-cols-2 gap-2">
+            <Button
+              color="ghost"
+              endIcon={<MagicWandFilled size={20} />}
+              className={`font-bold bg-black !rounded-full !bg-gradient-to-br ${BLUE_PINK_FILL} border-none text-white`}
+              onClick={handleTogglePreset}>
+              Surprise Me
+            </Button>
+            <Button
+              color="ghost"
+              className={`font-bold bg-black !rounded-full border-none text-white`}
+              startIcon={<ListBoxes size={20} />}
+              onClick={() => setShowEffectsCarousels(true)}>
+              More effects
+            </Button>
+          </div>
+        </div>
+      )}
+
+      <EffectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

demo/frontend/src/common/components/effects/MoreFunEffects.tsx

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {moreEffects} from '@/common/components/effects/EffectsUtils';
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {activeHighlightEffectAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function MoreFunEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeHighlightEffectAtom);
+
+  return (
+    <ToolbarSection title="Selected Objects" borderBottom={true}>
+      {moreEffects.map(effect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={effect.title}
+            icon={effect.Icon}
+            title={effect.title}
+            isActive={activeEffect.name === effect.effectName}
+            badge={
+              activeEffect.name === effect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              setEffect(effect.effectName, EffectIndex.HIGHLIGHT);
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}

demo/frontend/src/common/components/gallery/ChangeVideoModal.tsx

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import DemoVideoGalleryModal from '@/common/components/gallery/DemoVideoGalleryModal';
+import useVideo from '@/common/components/video/editor/useVideo';
+import Logger from '@/common/logger/Logger';
+import {isStreamingAtom, uploadingStateAtom, VideoData} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {ComponentType, useCallback} from 'react';
+import {useNavigate} from 'react-router-dom';
+
+type Props = {
+  videoGalleryModalTrigger?: ComponentType<VideoGalleryTriggerProps>;
+  showUploadInGallery?: boolean;
+  onChangeVideo?: () => void;
+};
+
+export default function ChangeVideoModal({
+  videoGalleryModalTrigger: VideoGalleryModalTriggerComponent,
+  showUploadInGallery = true,
+  onChangeVideo,
+}: Props) {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const video = useVideo();
+  const navigate = useNavigate();
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  function handlePauseOrAbortVideo() {
+    if (isStreaming) {
+      video?.abortStreamMasks();
+    } else {
+      handlePause();
+    }
+  }
+
+  function handleSwitchVideos(video: VideoData) {
+    // Retain any search parameter
+    navigate(
+      {
+        pathname: location.pathname,
+        search: location.search,
+      },
+      {
+        state: {
+          video,
+        },
+      },
+    );
+    onChangeVideo?.();
+  }
+
+  function handleUploadVideoError(error: Error) {
+    setUploadingState('error');
+    Logger.error(error);
+  }
+
+  return (
+    <DemoVideoGalleryModal
+      trigger={VideoGalleryModalTriggerComponent}
+      showUploadInGallery={showUploadInGallery}
+      onOpen={handlePauseOrAbortVideo}
+      onSelect={handleSwitchVideos}
+      onUploadVideoError={handleUploadVideoError}
+    />
+  );
+}

demo/frontend/src/common/components/gallery/DefaultVideoGalleryModalTrigger.tsx

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ResponsiveButton from '@/common/components/button/ResponsiveButton';
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import {ImageCopy} from '@carbon/icons-react';
+
+export default function DefaultVideoGalleryModalTrigger({
+  onClick,
+}: VideoGalleryTriggerProps) {
+  return (
+    <ResponsiveButton
+      color="ghost"
+      className="hover:!bg-black"
+      startIcon={<ImageCopy size={20} />}
+      onClick={onClick}>
+      Change video
+    </ResponsiveButton>
+  );
+}

demo/frontend/src/common/components/gallery/DemoVideoGallery.tsx

@@ -0,0 +1,209 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {DemoVideoGalleryQuery} from '@/common/components/gallery/__generated__/DemoVideoGalleryQuery.graphql';
+import VideoGalleryUploadVideo from '@/common/components/gallery/VideoGalleryUploadPhoto';
+import VideoPhoto from '@/common/components/gallery/VideoPhoto';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {VideoData} from '@/demo/atoms';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {fontSize, fontWeight, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+import PhotoAlbum, {Photo, RenderPhotoProps} from 'react-photo-album';
+import {graphql, useLazyLoadQuery} from 'react-relay';
+import {useLocation, useNavigate} from 'react-router-dom';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    marginHorizontal: spacing[1],
+    height: '100%',
+    lineHeight: 1.2,
+    paddingTop: spacing[8],
+  },
+  headerContainer: {
+    marginBottom: spacing[8],
+    fontWeight: fontWeight['medium'],
+    fontSize: fontSize['2xl'],
+    '@media screen and (max-width: 768px)': {
+      marginTop: spacing[0],
+      marginBottom: spacing[8],
+      marginHorizontal: spacing[4],
+      fontSize: fontSize['xl'],
+    },
+  },
+  albumContainer: {
+    flex: '1 1 0%',
+    width: '100%',
+    overflowY: 'auto',
+  },
+});
+
+type Props = {
+  showUploadInGallery?: boolean;
+  onSelect?: (video: VideoPhotoData) => void;
+  onUpload: (video: VideoData) => void;
+  onUploadStart?: () => void;
+  onUploadError?: (error: Error) => void;
+};
+
+type VideoPhotoData = Photo &
+  VideoData & {
+    poster: string;
+    isUploadOption: boolean;
+  };
+
+export default function DemoVideoGallery({
+  showUploadInGallery = false,
+  onSelect,
+  onUpload,
+  onUploadStart,
+  onUploadError,
+}: Props) {
+  const navigate = useNavigate();
+  const location = useLocation();
+  const {isMobile: isMobileScreenSize} = useScreenSize();
+
+  const data = useLazyLoadQuery<DemoVideoGalleryQuery>(
+    graphql`
+      query DemoVideoGalleryQuery {
+        videos {
+          edges {
+            node {
+              id
+              path
+              posterPath
+              url
+              posterUrl
+              height
+              width
+              posterUrl
+            }
+          }
+        }
+      }
+    `,
+    {},
+  );
+
+  const allVideos: VideoPhotoData[] = useMemo(() => {
+    return data.videos.edges.map(video => {
+      return {
+        src: video.node.url,
+        path: video.node.path,
+        poster: video.node.posterPath,
+        posterPath: video.node.posterPath,
+        url: video.node.url,
+        posterUrl: video.node.posterUrl,
+        width: video.node.width,
+        height: video.node.height,
+        isUploadOption: false,
+      } as VideoPhotoData;
+    });
+  }, [data.videos.edges]);
+
+  const shareableVideos: VideoPhotoData[] = useMemo(() => {
+    const filteredVideos = [...allVideos];
+
+    if (showUploadInGallery) {
+      const uploadOption = {
+        src: '',
+        width: 1280,
+        height: 720,
+        poster: '',
+        isUploadOption: true,
+      } as VideoPhotoData;
+      filteredVideos.unshift(uploadOption);
+    }
+
+    return filteredVideos;
+  }, [allVideos, showUploadInGallery]);
+
+  const renderPhoto = ({
+    photo: video,
+    imageProps,
+  }: RenderPhotoProps<VideoPhotoData>) => {
+    const {style} = imageProps;
+    const {url, posterUrl} = video;
+
+    return video.isUploadOption ? (
+      <VideoGalleryUploadVideo
+        style={style}
+        onUpload={handleUploadVideo}
+        onUploadError={onUploadError}
+        onUploadStart={onUploadStart}
+      />
+    ) : (
+      <VideoPhoto
+        src={url}
+        poster={posterUrl}
+        style={style}
+        onClick={() => {
+          navigate(location.pathname, {
+            state: {
+              video,
+            },
+          });
+          onSelect?.(video);
+        }}
+      />
+    );
+  };
+
+  function handleUploadVideo(video: VideoData) {
+    navigate(location.pathname, {
+      state: {
+        video,
+      },
+    });
+    onUpload?.(video);
+  }
+
+  const descriptionStyle = 'text-sm md:text-base text-gray-400 leading-snug';
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.albumContainer)}>
+        <div className="pt-0 md:px-16 md:pt-8 md:pb-8">
+          <div {...stylex.props(styles.headerContainer)}>
+            <h3 className="mb-2">
+              Select a video to try{' '}
+              <span className="hidden md:inline">
+                with the {DEMO_SHORT_NAME}
+              </span>
+            </h3>
+            <p className={descriptionStyle}>
+              You’ll be able to download what you make.
+            </p>
+          </div>
+
+          <PhotoAlbum<VideoPhotoData>
+            layout="rows"
+            photos={shareableVideos}
+            targetRowHeight={isMobileScreenSize ? 120 : 200}
+            rowConstraints={{
+              singleRowMaxHeight: isMobileScreenSize ? 120 : 240,
+              maxPhotos: 3,
+            }}
+            renderPhoto={renderPhoto}
+            spacing={4}
+          />
+        </div>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/gallery/DemoVideoGalleryModal.tsx

@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import DefaultVideoGalleryModalTrigger from '@/common/components/gallery/DefaultVideoGalleryModalTrigger';
+import {
+  frameIndexAtom,
+  sessionAtom,
+  uploadingStateAtom,
+  VideoData,
+} from '@/demo/atoms';
+import {spacing} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {useSetAtom} from 'jotai';
+import {ComponentType, useCallback, useRef} from 'react';
+import {Modal} from 'react-daisyui';
+import DemoVideoGallery from './DemoVideoGallery';
+
+const styles = stylex.create({
+  container: {
+    position: 'relative',
+    minWidth: '85vw',
+    minHeight: '85vh',
+    overflow: 'hidden',
+    color: '#fff',
+    boxShadow: '0 0 100px 50px #000',
+    borderRadius: 16,
+    border: '2px solid transparent',
+    background:
+      'linear-gradient(#1A1C1F, #1A1C1F) padding-box, linear-gradient(to right bottom, #FB73A5,#595FEF,#94EAE2,#FCCB6B) border-box',
+  },
+  closeButton: {
+    position: 'absolute',
+    top: 0,
+    right: 0,
+    padding: spacing[3],
+    zIndex: 10,
+    cursor: 'pointer',
+    ':hover': {
+      opacity: 0.7,
+    },
+  },
+  galleryContainer: {
+    position: 'absolute',
+    top: spacing[4],
+    left: 0,
+    right: 0,
+    bottom: 0,
+    overflowY: 'auto',
+  },
+});
+
+export type VideoGalleryTriggerProps = {
+  onClick: () => void;
+};
+
+type Props = {
+  trigger?: ComponentType<VideoGalleryTriggerProps>;
+  showUploadInGallery?: boolean;
+  onOpen?: () => void;
+  onSelect?: (video: VideoData, isUpload?: boolean) => void;
+  onUploadVideoError?: (error: Error) => void;
+};
+
+export default function DemoVideoGalleryModal({
+  trigger: VideoGalleryModalTrigger = DefaultVideoGalleryModalTrigger,
+  showUploadInGallery = false,
+  onOpen,
+  onSelect,
+  onUploadVideoError,
+}: Props) {
+  const modalRef = useRef<HTMLDialogElement | null>(null);
+
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const setSession = useSetAtom(sessionAtom);
+
+  function openModal() {
+    const modal = modalRef.current;
+    if (modal != null) {
+      modal.style.display = 'grid';
+      modal.showModal();
+    }
+  }
+
+  function closeModal() {
+    const modal = modalRef.current;
+    if (modal != null) {
+      modal.close();
+      modal.style.display = 'none';
+    }
+  }
+
+  const handleSelect = useCallback(
+    async (video: VideoData, isUpload?: boolean) => {
+      closeModal();
+      setFrameIndex(0);
+      onSelect?.(video, isUpload);
+      setUploadingState('default');
+      setSession(null);
+    },
+    [setFrameIndex, onSelect, setUploadingState, setSession],
+  );
+
+  function handleUploadVideoStart() {
+    setUploadingState('uploading');
+    closeModal();
+  }
+
+  function handleOpenVideoGalleryModal() {
+    onOpen?.();
+    openModal();
+  }
+
+  return (
+    <>
+      <VideoGalleryModalTrigger onClick={handleOpenVideoGalleryModal} />
+      <Modal ref={modalRef} {...stylex.props(styles.container)}>
+        <div onClick={closeModal} {...stylex.props(styles.closeButton)}>
+          <Close size={28} />
+        </div>
+        <Modal.Body>
+          <div {...stylex.props(styles.galleryContainer)}>
+            <DemoVideoGallery
+              showUploadInGallery={showUploadInGallery}
+              onSelect={video => handleSelect(video)}
+              onUpload={video => handleSelect(video, true)}
+              onUploadStart={handleUploadVideoStart}
+              onUploadError={onUploadVideoError}
+            />
+          </div>
+        </Modal.Body>
+      </Modal>
+    </>
+  );
+}

demo/frontend/src/common/components/gallery/VideoGalleryUploadPhoto.tsx

@@ -0,0 +1,102 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useUploadVideo from '@/common/components/gallery/useUploadVideo';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {VideoData} from '@/demo/atoms';
+import {MAX_UPLOAD_FILE_SIZE} from '@/demo/DemoConfig';
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+import {RetryFailed, Upload} from '@carbon/icons-react';
+import {CSSProperties, ReactNode} from 'react';
+import {Loading} from 'react-daisyui';
+
+type Props = {
+  style: CSSProperties;
+  onUpload: (video: VideoData) => void;
+  onUploadStart?: () => void;
+  onUploadError?: (error: Error) => void;
+};
+
+export default function VideoGalleryUploadVideo({
+  style,
+  onUpload,
+  onUploadStart,
+  onUploadError,
+}: Props) {
+  const {getRootProps, getInputProps, isUploading, error} = useUploadVideo({
+    onUpload,
+    onUploadStart,
+    onUploadError,
+  });
+  const {isMobile} = useScreenSize();
+
+  return (
+    <div className={`cursor-pointer ${BLUE_PINK_FILL_BR}`} style={style}>
+      <span {...getRootProps()}>
+        <input {...getInputProps()} />
+        <div className="relative w-full h-full">
+          <div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2">
+            {isUploading && (
+              <IconWrapper
+                icon={
+                  <Loading
+                    size={isMobile ? 'md' : 'lg'}
+                    className="text-white"
+                  />
+                }
+                title="Uploading ..."
+              />
+            )}
+            {error !== null && (
+              <IconWrapper
+                icon={<RetryFailed color="white" size={isMobile ? 24 : 32} />}
+                title={error}
+              />
+            )}
+            {!isUploading && error === null && (
+              <IconWrapper
+                icon={<Upload color="white" size={isMobile ? 24 : 32} />}
+                title={
+                  <>
+                    Upload{' '}
+                    <div className="text-xs opacity-70">
+                      Max {MAX_UPLOAD_FILE_SIZE}
+                    </div>
+                  </>
+                }
+              />
+            )}
+          </div>
+        </div>
+      </span>
+    </div>
+  );
+}
+
+type IconWrapperProps = {
+  icon: ReactNode;
+  title: ReactNode | string;
+};
+
+function IconWrapper({icon, title}: IconWrapperProps) {
+  return (
+    <>
+      <div className="flex justify-center">{icon}</div>
+      <div className="mt-1 text-sm md:text-lg text-white font-medium text-center leading-tight">
+        {title}
+      </div>
+    </>
+  );
+}

demo/frontend/src/common/components/gallery/VideoPhoto.tsx

@@ -0,0 +1,112 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Logger from '@/common/logger/Logger';
+import stylex from '@stylexjs/stylex';
+import {
+  CSSProperties,
+  MouseEventHandler,
+  useCallback,
+  useEffect,
+  useRef,
+} from 'react';
+
+const styles = stylex.create({
+  background: {
+    backgroundRepeat: 'no-repeat',
+    backgroundSize: 'cover',
+    backgroundPosition: 'center',
+    cursor: 'pointer',
+  },
+  video: {
+    width: '100%',
+    height: '100%',
+  },
+});
+
+type Props = {
+  onClick: MouseEventHandler<HTMLVideoElement> | undefined;
+  src: string;
+  poster: string;
+  style: CSSProperties;
+};
+
+export default function VideoPhoto({src, poster, style, onClick}: Props) {
+  const videoRef = useRef<HTMLVideoElement | null>(null);
+  const playPromiseRef = useRef<Promise<void> | null>(null);
+
+  const play = useCallback(() => {
+    const video = videoRef.current;
+    // Only play video if it is not already playing
+    if (video != null && video.paused) {
+      // This quirky way of handling video play/pause in the browser is needed
+      // due to the async nature of the video play API:
+      // https://developer.chrome.com/blog/play-request-was-interrupted/
+      const playPromise = video.play();
+      playPromise.catch(error => {
+        Logger.error('Failed to play video', error);
+      });
+      playPromiseRef.current = playPromise;
+    }
+  }, []);
+
+  const pause = useCallback(() => {
+    // Only pause video if it is playing
+    const playPromise = playPromiseRef.current;
+    if (playPromise != null) {
+      playPromise
+        .then(() => {
+          videoRef.current?.pause();
+        })
+        .catch(error => {
+          Logger.error('Failed to pause video', error);
+        })
+        .finally(() => {
+          playPromiseRef.current = null;
+        });
+    }
+  }, []);
+
+  useEffect(() => {
+    return () => {
+      pause();
+    };
+  }, [pause]);
+
+  return (
+    <div
+      style={{
+        ...style,
+        backgroundImage: `url(${poster})`,
+      }}
+      {...stylex.props(styles.background)}>
+      <video
+        ref={videoRef}
+        {...stylex.props(styles.video)}
+        preload="none"
+        playsInline
+        loop
+        muted
+        title="Gallery Video"
+        poster={poster}
+        onMouseEnter={play}
+        onMouseLeave={pause}
+        onClick={onClick}>
+        <source src={src} type="video/mp4" />
+        Sorry, your browser does not support embedded videos.
+      </video>
+    </div>
+  );
+}

demo/frontend/src/common/components/gallery/__generated__/DemoVideoGalleryModalQuery.graphql.ts

@@ -0,0 +1,303 @@
+/**
+ * @generated SignedSource<<db7e183e1996cf656749b4e33c2424e6>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Query } from 'relay-runtime';
+import { FragmentRefs } from "relay-runtime";
+export type DemoVideoGalleryModalQuery$variables = Record<PropertyKey, never>;
+export type DemoVideoGalleryModalQuery$data = {
+  readonly " $fragmentSpreads": FragmentRefs<"DatasetsDropdown_datasets" | "VideoGallery_videos">;
+};
+export type DemoVideoGalleryModalQuery = {
+  response: DemoVideoGalleryModalQuery$data;
+  variables: DemoVideoGalleryModalQuery$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "alias": null,
+    "args": null,
+    "kind": "ScalarField",
+    "name": "name",
+    "storageKey": null
+  }
+],
+v1 = [
+  {
+    "kind": "Literal",
+    "name": "after",
+    "value": ""
+  },
+  {
+    "kind": "Literal",
+    "name": "first",
+    "value": 20
+  }
+],
+v2 = {
+  "alias": null,
+  "args": null,
+  "kind": "ScalarField",
+  "name": "__typename",
+  "storageKey": null
+};
+return {
+  "fragment": {
+    "argumentDefinitions": [],
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "DemoVideoGalleryModalQuery",
+    "selections": [
+      {
+        "args": null,
+        "kind": "FragmentSpread",
+        "name": "DatasetsDropdown_datasets"
+      },
+      {
+        "args": null,
+        "kind": "FragmentSpread",
+        "name": "VideoGallery_videos"
+      }
+    ],
+    "type": "Query",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": [],
+    "kind": "Operation",
+    "name": "DemoVideoGalleryModalQuery",
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "DatasetConnection",
+        "kind": "LinkedField",
+        "name": "datasets",
+        "plural": false,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "DatasetEdge",
+            "kind": "LinkedField",
+            "name": "edges",
+            "plural": true,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "concreteType": "Dataset",
+                "kind": "LinkedField",
+                "name": "node",
+                "plural": false,
+                "selections": (v0/*: any*/),
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": (v1/*: any*/),
+        "concreteType": "VideoConnection",
+        "kind": "LinkedField",
+        "name": "videos",
+        "plural": false,
+        "selections": [
+          (v2/*: any*/),
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "PageInfo",
+            "kind": "LinkedField",
+            "name": "pageInfo",
+            "plural": false,
+            "selections": [
+              (v2/*: any*/),
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "hasPreviousPage",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "hasNextPage",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "startCursor",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "endCursor",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          },
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "VideoEdge",
+            "kind": "LinkedField",
+            "name": "edges",
+            "plural": true,
+            "selections": [
+              (v2/*: any*/),
+              {
+                "alias": null,
+                "args": null,
+                "concreteType": "Video",
+                "kind": "LinkedField",
+                "name": "node",
+                "plural": false,
+                "selections": [
+                  (v2/*: any*/),
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "id",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "path",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "posterPath",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "url",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "posterUrl",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "width",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "height",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "concreteType": "Dataset",
+                    "kind": "LinkedField",
+                    "name": "dataset",
+                    "plural": false,
+                    "selections": (v0/*: any*/),
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "concreteType": "VideoPermissions",
+                    "kind": "LinkedField",
+                    "name": "permissions",
+                    "plural": false,
+                    "selections": [
+                      {
+                        "alias": null,
+                        "args": null,
+                        "kind": "ScalarField",
+                        "name": "canShare",
+                        "storageKey": null
+                      },
+                      {
+                        "alias": null,
+                        "args": null,
+                        "kind": "ScalarField",
+                        "name": "canDownload",
+                        "storageKey": null
+                      }
+                    ],
+                    "storageKey": null
+                  }
+                ],
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "cursor",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": "videos(after:\"\",first:20)"
+      },
+      {
+        "alias": null,
+        "args": (v1/*: any*/),
+        "filters": [
+          "datasetName"
+        ],
+        "handle": "connection",
+        "key": "VideoGallery_videos",
+        "kind": "LinkedHandle",
+        "name": "videos"
+      }
+    ]
+  },
+  "params": {
+    "cacheID": "e0bccf553377682e6bc283c2ce53bee5",
+    "id": null,
+    "metadata": {},
+    "name": "DemoVideoGalleryModalQuery",
+    "operationKind": "query",
+    "text": "query DemoVideoGalleryModalQuery {\n  ...DatasetsDropdown_datasets\n  ...VideoGallery_videos\n}\n\nfragment DatasetsDropdown_datasets on Query {\n  datasets {\n    edges {\n      node {\n        name\n      }\n    }\n  }\n}\n\nfragment VideoGallery_videos on Query {\n  videos(first: 20, after: \"\") {\n    __typename\n    pageInfo {\n      __typename\n      hasPreviousPage\n      hasNextPage\n      startCursor\n      endCursor\n    }\n    edges {\n      __typename\n      node {\n        __typename\n        id\n        path\n        posterPath\n        url\n        posterUrl\n        width\n        height\n        dataset {\n          name\n        }\n        permissions {\n          canShare\n          canDownload\n        }\n      }\n      cursor\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "d09e34e2b9f2e25c2d564106de5f9c89";
+
+export default node;

demo/frontend/src/common/components/gallery/__generated__/DemoVideoGalleryQuery.graphql.ts

@@ -0,0 +1,148 @@
+/**
+ * @generated SignedSource<<20d31a82b5f3b251b0e42b4f0e3522b8>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Query } from 'relay-runtime';
+export type DemoVideoGalleryQuery$variables = Record<PropertyKey, never>;
+export type DemoVideoGalleryQuery$data = {
+  readonly videos: {
+    readonly edges: ReadonlyArray<{
+      readonly node: {
+        readonly height: number;
+        readonly id: any;
+        readonly path: string;
+        readonly posterPath: string | null | undefined;
+        readonly posterUrl: string;
+        readonly url: string;
+        readonly width: number;
+      };
+    }>;
+  };
+};
+export type DemoVideoGalleryQuery = {
+  response: DemoVideoGalleryQuery$data;
+  variables: DemoVideoGalleryQuery$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "alias": null,
+    "args": null,
+    "concreteType": "VideoConnection",
+    "kind": "LinkedField",
+    "name": "videos",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "VideoEdge",
+        "kind": "LinkedField",
+        "name": "edges",
+        "plural": true,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "Video",
+            "kind": "LinkedField",
+            "name": "node",
+            "plural": false,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "id",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "path",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "posterPath",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "url",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "posterUrl",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "height",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "width",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": [],
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "DemoVideoGalleryQuery",
+    "selections": (v0/*: any*/),
+    "type": "Query",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": [],
+    "kind": "Operation",
+    "name": "DemoVideoGalleryQuery",
+    "selections": (v0/*: any*/)
+  },
+  "params": {
+    "cacheID": "4dae74153a5528f2631b59dfb0adb021",
+    "id": null,
+    "metadata": {},
+    "name": "DemoVideoGalleryQuery",
+    "operationKind": "query",
+    "text": "query DemoVideoGalleryQuery {\n  videos {\n    edges {\n      node {\n        id\n        path\n        posterPath\n        url\n        posterUrl\n        height\n        width\n      }\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "d22ac5e58f6e4eb696651be49b410e4e";
+
+export default node;

demo/frontend/src/common/components/gallery/__generated__/useUploadVideoMutation.graphql.ts

@@ -0,0 +1,137 @@
+/**
+ * @generated SignedSource<<76014dced98d6c8989e7322712e38963>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type useUploadVideoMutation$variables = {
+  file: any;
+};
+export type useUploadVideoMutation$data = {
+  readonly uploadVideo: {
+    readonly height: number;
+    readonly id: any;
+    readonly path: string;
+    readonly posterPath: string | null | undefined;
+    readonly posterUrl: string;
+    readonly url: string;
+    readonly width: number;
+  };
+};
+export type useUploadVideoMutation = {
+  response: useUploadVideoMutation$data;
+  variables: useUploadVideoMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "file"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "file",
+        "variableName": "file"
+      }
+    ],
+    "concreteType": "Video",
+    "kind": "LinkedField",
+    "name": "uploadVideo",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "id",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "height",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "width",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "url",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "path",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "posterPath",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "posterUrl",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "useUploadVideoMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "useUploadVideoMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "dcbaf1bf411627fdb9dfbb827592cfc0",
+    "id": null,
+    "metadata": {},
+    "name": "useUploadVideoMutation",
+    "operationKind": "mutation",
+    "text": "mutation useUploadVideoMutation(\n  $file: Upload!\n) {\n  uploadVideo(file: $file) {\n    id\n    height\n    width\n    url\n    path\n    posterPath\n    posterUrl\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "710e462504d76597af8695b7fc70b4cf";
+
+export default node;

demo/frontend/src/common/components/gallery/useUploadVideo.ts

@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useUploadVideoMutation} from '@/common/components/gallery/__generated__/useUploadVideoMutation.graphql';
+import Logger from '@/common/logger/Logger';
+import {VideoData} from '@/demo/atoms';
+import {useState} from 'react';
+import {FileRejection, FileWithPath, useDropzone} from 'react-dropzone';
+import {graphql, useMutation} from 'react-relay';
+
+const ACCEPT_VIDEOS = {
+  'video/mp4': ['.mp4'],
+  'video/quicktime': ['.mov'],
+};
+
+// 70 MB default max video upload size
+const MAX_FILE_SIZE_IN_MB = 70;
+const MAX_VIDEO_UPLOAD_SIZE = MAX_FILE_SIZE_IN_MB * 1024 ** 2;
+
+type Props = {
+  onUpload: (video: VideoData) => void;
+  onUploadStart?: () => void;
+  onUploadError?: (error: Error) => void;
+};
+
+export default function useUploadVideo({
+  onUpload,
+  onUploadStart,
+  onUploadError,
+}: Props) {
+  const [error, setError] = useState<string | null>(null);
+  const [commit, isMutationInFlight] = useMutation<useUploadVideoMutation>(
+    graphql`
+      mutation useUploadVideoMutation($file: Upload!) {
+        uploadVideo(file: $file) {
+          id
+          height
+          width
+          url
+          path
+          posterPath
+          posterUrl
+        }
+      }
+    `,
+  );
+
+  const {getRootProps, getInputProps} = useDropzone({
+    accept: ACCEPT_VIDEOS,
+    multiple: false,
+    maxFiles: 1,
+    onDrop: (
+      acceptedFiles: FileWithPath[],
+      fileRejections: FileRejection[],
+    ) => {
+      setError(null);
+
+      // Check if any of the files (only 1 file allowed) is rejected. The
+      // rejected file has an error (e.g., 'file-too-large'). Rendering an
+      // appropriate message.
+      if (fileRejections.length > 0 && fileRejections[0].errors.length > 0) {
+        const code = fileRejections[0].errors[0].code;
+        if (code === 'file-too-large') {
+          setError(
+            `File too large. Try a video under ${MAX_FILE_SIZE_IN_MB} MB`,
+          );
+          return;
+        }
+      }
+
+      if (acceptedFiles.length === 0) {
+        setError('File not accepted. Please try again.');
+        return;
+      }
+      if (acceptedFiles.length > 1) {
+        setError('Too many files. Please try again with 1 file.');
+        return;
+      }
+
+      onUploadStart?.();
+      const file = acceptedFiles[0];
+
+      commit({
+        variables: {
+          file,
+        },
+        uploadables: {
+          file,
+        },
+        onCompleted: response => onUpload(response.uploadVideo),
+        onError: error => {
+          Logger.error(error);
+          onUploadError?.(error);
+          setError('Upload failed.');
+        },
+      });
+    },
+    onError: error => {
+      Logger.error(error);
+      setError('File not supported.');
+    },
+    maxSize: MAX_VIDEO_UPLOAD_SIZE,
+  });
+
+  return {
+    getRootProps,
+    getInputProps,
+    isUploading: isMutationInFlight,
+    error,
+    setError,
+  };
+}

demo/frontend/src/common/components/icons/GitHubIcon.tsx

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type Props = {
+  className?: string;
+};
+
+export function GitHubIcon({className}: Props) {
+  return (
+    <svg viewBox="0 0 24 24" aria-hidden="true" className={className}>
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M12 2C6.477 2 2 6.463 2 11.97c0 4.404 2.865 8.14 6.839 9.458.5.092.682-.216.682-.48 0-.236-.008-.864-.013-1.695-2.782.602-3.369-1.337-3.369-1.337-.454-1.151-1.11-1.458-1.11-1.458-.908-.618.069-.606.069-.606 1.003.07 1.531 1.027 1.531 1.027.892 1.524 2.341 1.084 2.91.828.092-.643.35-1.083.636-1.332-2.22-.251-4.555-1.107-4.555-4.927 0-1.088.39-1.979 1.029-2.675-.103-.252-.446-1.266.098-2.638 0 0 .84-.268 2.75 1.022A9.607 9.607 0 0 1 12 6.82c.85.004 1.705.114 2.504.336 1.909-1.29 2.747-1.022 2.747-1.022.546 1.372.202 2.386.1 2.638.64.696 1.028 1.587 1.028 2.675 0 3.83-2.339 4.673-4.566 4.92.359.307.678.915.678 1.846 0 1.332-.012 2.407-.012 2.734 0 .267.18.577.688.48 3.97-1.32 6.833-5.054 6.833-9.458C22 6.463 17.522 2 12 2Z"></path>
+    </svg>
+  );
+}

demo/frontend/src/common/components/options/DownloadOption.tsx

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Package} from '@carbon/icons-react';
+import OptionButton from './OptionButton';
+import useDownloadVideo from './useDownloadVideo';
+
+export default function DownloadOption() {
+  const {download, state} = useDownloadVideo();
+
+  return (
+    <OptionButton
+      title="Download"
+      Icon={Package}
+      loadingProps={{
+        loading: state === 'started' || state === 'encoding',
+        label: 'Downloading...',
+      }}
+      onClick={download}
+    />
+  );
+}

demo/frontend/src/common/components/options/GalleryOption.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideoModal from '@/common/components/gallery/ChangeVideoModal';
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {ImageCopy} from '@carbon/icons-react';
+import OptionButton from './OptionButton';
+
+type Props = {
+  onChangeVideo: () => void;
+};
+export default function GalleryOption({onChangeVideo}: Props) {
+  return (
+    <ChangeVideoModal
+      videoGalleryModalTrigger={GalleryTrigger}
+      showUploadInGallery={false}
+      onChangeVideo={onChangeVideo}
+    />
+  );
+}
+
+function GalleryTrigger({onClick}: VideoGalleryTriggerProps) {
+  const {isMobile} = useScreenSize();
+
+  return (
+    <OptionButton
+      variant="flat"
+      title={isMobile ? 'Gallery' : 'Browse gallery'}
+      Icon={ImageCopy}
+      onClick={onClick}
+    />
+  );
+}

demo/frontend/src/common/components/options/MoreOptionsToolbar.tsx

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import MoreOptionsToolbarBottomActions from '@/common/components/options/MoreOptionsToolbarBottomActions';
+import ShareSection from '@/common/components/options/ShareSection';
+import TryAnotherVideoSection from '@/common/components/options/TryAnotherVideoSection';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {useEffect, useRef} from 'react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MoreOptionsToolbar({onTabChange}: Props) {
+  const {isMobile} = useScreenSize();
+  const {clearMessage} = useMessagesSnackbar();
+  const didClearMessageSnackbar = useRef(false);
+
+  useEffect(() => {
+    if (!didClearMessageSnackbar.current) {
+      didClearMessageSnackbar.current = true;
+      clearMessage();
+    }
+  }, [clearMessage]);
+
+  return (
+    <div className="flex flex-col h-full">
+      <div className="grow">
+        <ToolbarHeaderWrapper
+          title="Nice work! What's next?"
+          className="pb-0 !border-b-0 !text-white"
+          showProgressChip={false}
+        />
+        <ShareSection />
+        {!isMobile && <div className="h-[1px] bg-black mt-4 mb-8"></div>}
+        <TryAnotherVideoSection onTabChange={onTabChange} />
+      </div>
+      {!isMobile && (
+        <MoreOptionsToolbarBottomActions onTabChange={onTabChange} />
+      )}
+    </div>
+  );
+}

demo/frontend/src/common/components/options/MoreOptionsToolbarBottomActions.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import RestartSessionButton from '@/common/components/session/RestartSessionButton';
+import {
+  EFFECT_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {ChevronLeft} from '@carbon/icons-react';
+import {Button} from 'react-daisyui';
+import ToolbarBottomActionsWrapper from '../toolbar/ToolbarBottomActionsWrapper';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MoreOptionsToolbarBottomActions({onTabChange}: Props) {
+  function handleReturnToEffectsTab() {
+    onTabChange(EFFECT_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <Button
+        color="ghost"
+        onClick={handleReturnToEffectsTab}
+        className="!px-4 !rounded-full font-medium text-white hover:bg-black"
+        startIcon={<ChevronLeft />}>
+        Edit effects
+      </Button>
+      <RestartSessionButton
+        onRestartSession={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+    </ToolbarBottomActionsWrapper>
+  );
+}

demo/frontend/src/common/components/options/OptionButton.tsx

@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {Loading} from 'react-daisyui';
+
+type Props = {
+  variant?: 'default' | 'flat' | 'gradient';
+  title: string | React.ReactNode;
+  Icon: CarbonIconType;
+  isActive?: boolean;
+  isDisabled?: boolean;
+  loadingProps?: {
+    loading: boolean;
+    label?: string;
+  };
+  onClick: () => void;
+};
+
+export default function OptionButton({
+  variant = 'default',
+  title,
+  Icon,
+  isActive = false,
+  isDisabled = false,
+  loadingProps,
+  onClick,
+}: Props) {
+  const {isMobile} = useScreenSize();
+  const isLoading = loadingProps?.loading === true;
+
+  function handleClick() {
+    if (isDisabled) {
+      return;
+    }
+    onClick();
+  }
+
+  const ButtonBase = (
+    <div
+      onClick={handleClick}
+      className={`relative rounded-lg h-full flex items-center justify-center 
+${variant === 'default' ? 'bg-graydark-700' : ''}
+${!isDisabled && 'cursor-pointer'}
+${isDisabled ? 'text-gray-300' : ''}
+${isActive && BLUE_PINK_FILL_BR}`}>
+      <div className="flex gap-2 items-center py-4 md:py-6">
+        {isLoading ? (
+          <Loading size="md" className="mx-auto mt-1" />
+        ) : (
+          <Icon
+            size={isMobile ? 24 : 28}
+            className={`mx-auto ${isDisabled ? 'text-gray-300' : 'text-white'}`}
+          />
+        )}
+        <div className="text-base font-medium text-white">
+          {isLoading && loadingProps?.label != null
+            ? loadingProps.label
+            : title}
+        </div>
+      </div>
+    </div>
+  );
+
+  return variant === 'gradient' ? (
+    <GradientBorder rounded={false} className={'rounded-lg md:rounded-full'}>
+      {ButtonBase}
+    </GradientBorder>
+  ) : (
+    ButtonBase
+  );
+}

demo/frontend/src/common/components/options/ShareSection.tsx

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import DownloadOption from './DownloadOption';
+
+export default function ShareSection() {
+  return (
+    <div className="p-5 md:p-8">
+      <DownloadOption />
+    </div>
+  );
+}

demo/frontend/src/common/components/options/ShareUtils.ts

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export async function handleSaveVideo(
+  videoPath: string,
+  fileName?: string,
+): Promise<void> {
+  const blob = await fetch(videoPath).then(res => res.blob());
+
+  return new Promise(resolve => {
+    const reader = new FileReader();
+    reader.readAsDataURL(blob);
+    reader.addEventListener('load', () => {
+      const elem = document.createElement('a');
+      elem.download = fileName ?? getFileName();
+      if (typeof reader.result === 'string') {
+        elem.href = reader.result;
+      }
+      elem.click();
+      resolve();
+    });
+  });
+}
+
+export function getFileName() {
+  const date = new Date();
+  const timestamp = date.getTime();
+  return `sam2_masked_video_${timestamp}.mp4`;
+}

demo/frontend/src/common/components/options/TryAnotherVideoSection.tsx

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GalleryOption from '@/common/components/options/GalleryOption';
+import UploadOption from '@/common/components/options/UploadOption';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useScreenSize from '@/common/screen/useScreenSize';
+
+type Props = {
+  onTabChange: (tabIndex: number) => void;
+};
+
+export default function TryAnotherVideoSection({onTabChange}: Props) {
+  const {isMobile} = useScreenSize();
+  const video = useVideo();
+
+  function handleVideoChange() {
+    if (video != null) {
+      video.pause();
+      video.frame = 0;
+    }
+    onTabChange(OBJECT_TOOLBAR_INDEX);
+  }
+
+  if (isMobile) {
+    return (
+      <div className="px-8 pb-8">
+        <div className="font-medium text-gray-300 text-sm">
+          Or, try another video
+        </div>
+        <div className="flex flex-row gap-4 mt-4 w-full">
+          <div className="flex-1">
+            <UploadOption onUpload={handleVideoChange} />
+          </div>
+          <div className="flex-1">
+            <GalleryOption onChangeVideo={handleVideoChange} />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
+  return (
+    <div className="px-8 pb-8">
+      <div className="font-medium text-gray-300 text-base">
+        Try another video
+      </div>
+      <div className="flex flex-col gap-4 mt-4">
+        <UploadOption onUpload={handleVideoChange} />
+        <GalleryOption onChangeVideo={handleVideoChange} />
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/options/UploadOption.tsx

@@ -0,0 +1,86 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useUploadVideo from '@/common/components/gallery/useUploadVideo';
+import OptionButton from '@/common/components/options/OptionButton';
+import Logger from '@/common/logger/Logger';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {sessionAtom, uploadingStateAtom} from '@/demo/atoms';
+import {MAX_UPLOAD_FILE_SIZE} from '@/demo/DemoConfig';
+import {Close, CloudUpload} from '@carbon/icons-react';
+import {useSetAtom} from 'jotai';
+import {useNavigate} from 'react-router-dom';
+
+type Props = {
+  onUpload: () => void;
+};
+
+export default function UploadOption({onUpload}: Props) {
+  const navigate = useNavigate();
+  const {isMobile} = useScreenSize();
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const setSession = useSetAtom(sessionAtom);
+
+  const {getRootProps, getInputProps, isUploading, error} = useUploadVideo({
+    onUpload: videoData => {
+      navigate(
+        {pathname: location.pathname, search: location.search},
+        {state: {video: videoData}},
+      );
+      onUpload();
+      setUploadingState('default');
+      setSession(null);
+    },
+    onUploadError: (error: Error) => {
+      setUploadingState('error');
+      Logger.error(error);
+    },
+    onUploadStart: () => {
+      setUploadingState('uploading');
+    },
+  });
+
+  return (
+    <div className="cursor-pointer" {...getRootProps()}>
+      <input {...getInputProps()} />
+
+      <OptionButton
+        variant="gradient"
+        title={
+          error !== null ? (
+            'Upload Error'
+          ) : isMobile ? (
+            <>
+              Upload{' '}
+              <div className="text-xs opacity-70">
+                Max {MAX_UPLOAD_FILE_SIZE}
+              </div>
+            </>
+          ) : (
+            <>
+              Upload your own{' '}
+              <div className="text-xs opacity-70">
+                Max {MAX_UPLOAD_FILE_SIZE}
+              </div>
+            </>
+          )
+        }
+        Icon={error !== null ? Close : CloudUpload}
+        loadingProps={{loading: isUploading, label: 'Uploading...'}}
+        onClick={() => {}}
+      />
+    </div>
+  );
+}

demo/frontend/src/common/components/options/__generated__/GetLinkOptionShareVideoMutation.graphql.ts

@@ -0,0 +1,89 @@
+/**
+ * @generated SignedSource<<39d7e92a6c15de1583c90ae21a7825e5>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type GetLinkOptionShareVideoMutation$variables = {
+  file: any;
+};
+export type GetLinkOptionShareVideoMutation$data = {
+  readonly uploadSharedVideo: {
+    readonly path: string;
+  };
+};
+export type GetLinkOptionShareVideoMutation = {
+  response: GetLinkOptionShareVideoMutation$data;
+  variables: GetLinkOptionShareVideoMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "file"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "file",
+        "variableName": "file"
+      }
+    ],
+    "concreteType": "SharedVideo",
+    "kind": "LinkedField",
+    "name": "uploadSharedVideo",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "path",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "GetLinkOptionShareVideoMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "GetLinkOptionShareVideoMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "f02ec81a41c8d75c3733853e1fb04f58",
+    "id": null,
+    "metadata": {},
+    "name": "GetLinkOptionShareVideoMutation",
+    "operationKind": "mutation",
+    "text": "mutation GetLinkOptionShareVideoMutation(\n  $file: Upload!\n) {\n  uploadSharedVideo(file: $file) {\n    path\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "c1b085da9afaac5f19eeb99ff561ed55";
+
+export default node;

demo/frontend/src/common/components/options/useDownloadVideo.ts

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {getFileName} from '@/common/components/options/ShareUtils';
+import {
+  EncodingCompletedEvent,
+  EncodingStateUpdateEvent,
+} from '@/common/components/video/VideoWorkerBridge';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {MP4ArrayBuffer} from 'mp4box';
+import {useState} from 'react';
+
+type DownloadingState = 'default' | 'started' | 'encoding' | 'completed';
+
+type State = {
+  state: DownloadingState;
+  progress: number;
+  download: (shouldSave?: boolean) => Promise<MP4ArrayBuffer>;
+};
+
+export default function useDownloadVideo(): State {
+  const [downloadingState, setDownloadingState] =
+    useState<DownloadingState>('default');
+  const [progress, setProgress] = useState<number>(0);
+
+  const video = useVideo();
+
+  async function download(shouldSave = true): Promise<MP4ArrayBuffer> {
+    return new Promise(resolve => {
+      function onEncodingStateUpdate(event: EncodingStateUpdateEvent) {
+        setDownloadingState('encoding');
+        setProgress(event.progress);
+      }
+
+      function onEncodingComplete(event: EncodingCompletedEvent) {
+        const file = event.file;
+
+        if (shouldSave) {
+          saveVideo(file, getFileName());
+        }
+
+        video?.removeEventListener('encodingCompleted', onEncodingComplete);
+        video?.removeEventListener(
+          'encodingStateUpdate',
+          onEncodingStateUpdate,
+        );
+        setDownloadingState('completed');
+        resolve(file);
+      }
+
+      video?.addEventListener('encodingStateUpdate', onEncodingStateUpdate);
+      video?.addEventListener('encodingCompleted', onEncodingComplete);
+
+      if (downloadingState === 'default' || downloadingState === 'completed') {
+        setDownloadingState('started');
+        video?.pause();
+        video?.encode();
+      }
+    });
+  }
+
+  function saveVideo(file: MP4ArrayBuffer, fileName: string) {
+    const blob = new Blob([file]);
+    const url = window.URL.createObjectURL(blob);
+
+    const a = document.createElement('a');
+    document.body.appendChild(a);
+    a.setAttribute('href', url);
+    a.setAttribute('download', fileName);
+    a.setAttribute('target', '_self');
+    a.click();
+    window.URL.revokeObjectURL(url);
+  }
+
+  return {download, progress, state: downloadingState};
+}

demo/frontend/src/common/components/session/RestartSessionButton.tsx

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useRestartSession from '@/common/components/session/useRestartSession';
+import {Reset} from '@carbon/icons-react';
+import {Button, Loading} from 'react-daisyui';
+
+type Props = {
+  onRestartSession: () => void;
+};
+
+export default function RestartSessionButton({onRestartSession}: Props) {
+  const {restartSession, isLoading} = useRestartSession();
+
+  function handleRestartSession() {
+    restartSession(onRestartSession);
+  }
+
+  return (
+    <Button
+      color="ghost"
+      onClick={handleRestartSession}
+      className="!px-4 !rounded-full font-medium text-white hover:bg-black"
+      startIcon={isLoading ? <Loading size="sm" /> : <Reset size={20} />}>
+      Start over
+    </Button>
+  );
+}

demo/frontend/src/common/components/session/__generated__/useCloseSessionBeforeUnloadMutation.graphql.ts

@@ -0,0 +1,92 @@
+/**
+ * @generated SignedSource<<f56872c0a8b65fa7e9bdaff351930ff0>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type CloseSessionInput = {
+  sessionId: string;
+};
+export type useCloseSessionBeforeUnloadMutation$variables = {
+  input: CloseSessionInput;
+};
+export type useCloseSessionBeforeUnloadMutation$data = {
+  readonly closeSession: {
+    readonly success: boolean;
+  };
+};
+export type useCloseSessionBeforeUnloadMutation = {
+  response: useCloseSessionBeforeUnloadMutation$data;
+  variables: useCloseSessionBeforeUnloadMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "CloseSession",
+    "kind": "LinkedField",
+    "name": "closeSession",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "success",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "useCloseSessionBeforeUnloadMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "useCloseSessionBeforeUnloadMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "99b73bd43a9f74104d545778cebbd15c",
+    "id": null,
+    "metadata": {},
+    "name": "useCloseSessionBeforeUnloadMutation",
+    "operationKind": "mutation",
+    "text": "mutation useCloseSessionBeforeUnloadMutation(\n  $input: CloseSessionInput!\n) {\n  closeSession(input: $input) {\n    success\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "55dd870645c9736b797b90819ddb1b92";
+
+export default node;

demo/frontend/src/common/components/session/useCloseSessionBeforeUnload.ts

@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useCloseSessionBeforeUnloadMutation$variables} from '@/common/components/session/__generated__/useCloseSessionBeforeUnloadMutation.graphql';
+import {sessionAtom} from '@/demo/atoms';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {useAtomValue} from 'jotai';
+import {useEffect, useMemo} from 'react';
+import {ConcreteRequest, graphql} from 'relay-runtime';
+
+/**
+ * The useCloseSessionBeforeUnload is a dirty workaround to send close session
+ * requests on window/tab close. Going through Relay does not send the request
+ * even if the `keepalive` flag is set for the request. It does work when the
+ * fetch is called directly with the close session mutation.
+ *
+ * Caveat: there is static typing, but there might be other caveats around this
+ * quirky hack.
+ */
+export default function useCloseSessionBeforeUnload() {
+  const session = useAtomValue(sessionAtom);
+  const {settings} = useSettingsContext();
+
+  const data = useMemo(() => {
+    if (session == null) {
+      return null;
+    }
+
+    const graphQLTaggedNode = graphql`
+      mutation useCloseSessionBeforeUnloadMutation($input: CloseSessionInput!) {
+        closeSession(input: $input) {
+          success
+        }
+      }
+    ` as ConcreteRequest;
+
+    const variables: useCloseSessionBeforeUnloadMutation$variables = {
+      input: {
+        sessionId: session.id,
+      },
+    };
+
+    const query = graphQLTaggedNode.params.text;
+    if (query === null) {
+      return null;
+    }
+
+    return {
+      query,
+      variables,
+    };
+  }, [session]);
+
+  useEffect(() => {
+    function onBeforeUpload() {
+      if (data == null) {
+        return;
+      }
+
+      fetch(`${settings.inferenceAPIEndpoint}/graphql`, {
+        method: 'POST',
+        credentials: 'include',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        keepalive: true,
+        body: JSON.stringify(data),
+      });
+    }
+    window.addEventListener('beforeunload', onBeforeUpload);
+    return () => {
+      window.removeEventListener('beforeunload', onBeforeUpload);
+    };
+  }, [data, session, settings.inferenceAPIEndpoint]);
+}

demo/frontend/src/common/components/session/useRestartSession.ts

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useInputVideo from '@/common/components/video/useInputVideo';
+import {
+  activeTrackletObjectIdAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+  labelTypeAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {useState} from 'react';
+
+export default function useRestartSession() {
+  const [isLoading, setIsLoading] = useState<boolean>();
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setActiveTrackletObjectId = useSetAtom(activeTrackletObjectIdAtom);
+  const setTracklets = useSetAtom(trackletObjectsAtom);
+  const setLabelType = useSetAtom(labelTypeAtom);
+  const {clearMessage} = useMessagesSnackbar();
+
+  const {inputVideo} = useInputVideo();
+  const video = useVideo();
+
+  async function restartSession(onRestart?: () => void) {
+    if (video === null || inputVideo === null) {
+      return;
+    }
+
+    setIsLoading(true);
+    if (isPlaying) {
+      video.pause();
+    }
+    if (isStreaming) {
+      await video.abortStreamMasks();
+    }
+    await video?.startSession(inputVideo.path);
+    video.frame = 0;
+    setActiveTrackletObjectId(0);
+    setTracklets([]);
+    setLabelType('positive');
+    onRestart?.();
+    clearMessage();
+    setIsLoading(false);
+  }
+
+  return {isLoading, restartSession};
+}

demo/frontend/src/common/components/snackbar/DemoMessagesSnackbarUtils.ts

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {EnqueueOption} from '@/common/components/snackbar/useMessagesSnackbar';
+
+export type MessageOptions = EnqueueOption & {
+  repeat?: boolean;
+};
+
+type MessageEvent = {
+  text: string;
+  shown: boolean;
+  action?: Element;
+  options?: MessageOptions;
+};
+
+export interface MessagesEventMap {
+  startSession: MessageEvent;
+  firstClick: MessageEvent;
+  pointClick: MessageEvent;
+  addObjectClick: MessageEvent;
+  trackAndPlayClick: MessageEvent;
+  trackAndPlayComplete: MessageEvent;
+  trackAndPlayThrottlingWarning: MessageEvent;
+  effectsMessage: MessageEvent;
+}
+
+export const defaultMessageMap: MessagesEventMap = {
+  startSession: {
+    text: 'Starting session',
+    shown: false,
+    options: {type: 'loading', showClose: false, repeat: true, duration: 2000},
+  },
+  firstClick: {
+    text: 'Tip: Click on any object in the video to get started.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  pointClick: {
+    text: 'Tip: Not what you expected? Add a few more clicks until the full object you want is selected.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  addObjectClick: {
+    text: 'Tip: Add a new object by clicking on it in the video.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  trackAndPlayClick: {
+    text: 'Hang tight while your objects are tracked! You’ll be able to apply visual effects in the next step. Stop tracking at any point to adjust your selections if the tracking doesn’t look right.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  trackAndPlayComplete: {
+    text: 'Tip: You can fix tracking issues by going back to the frames where tracking is not quite right and adding or removing clicks.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  trackAndPlayThrottlingWarning: {
+    text: 'Looks like you have clicked the tracking button a bit too often! To keep things running smoothly, we have temporarily disabled the button.',
+    shown: false,
+    options: {repeat: true},
+  },
+  effectsMessage: {
+    text: 'Tip: If you aren’t sure where to get started, click “Surprise Me” to apply a surprise effect to your video.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+};

demo/frontend/src/common/components/snackbar/MessagesSnackbar.tsx

@@ -0,0 +1,118 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useScreenSize from '@/common/screen/useScreenSize';
+import {color, gradients} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue} from 'jotai';
+import {Loading, RadialProgress} from 'react-daisyui';
+import {messageAtom} from './snackbarAtoms';
+import useExpireMessage from './useExpireMessage';
+import useMessagesSnackbar from './useMessagesSnackbar';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    top: '8px',
+    right: '8px',
+  },
+  mobileContainer: {
+    position: 'absolute',
+    bottom: '8px',
+    left: '8px',
+    right: '8px',
+  },
+  messageContainer: {
+    padding: '20px 20px',
+    color: '#FFF',
+    borderRadius: '8px',
+    fontSize: '0.9rem',
+    maxWidth: 400,
+    border: '2px solid transparent',
+    background: gradients['yellowTeal'],
+  },
+  messageWarningContainer: {
+    background: '#FFDC32',
+    color: color['gray-900'],
+  },
+  messageContent: {
+    display: 'flex',
+    alignItems: 'center',
+    gap: '8px',
+  },
+  progress: {
+    flexShrink: 0,
+    color: 'rgba(255, 255, 255, 0.1)',
+  },
+  closeColumn: {
+    display: 'flex',
+    alignSelf: 'stretch',
+    alignItems: 'start',
+  },
+});
+
+export default function MessagesSnackbar() {
+  const message = useAtomValue(messageAtom);
+  const {clearMessage} = useMessagesSnackbar();
+  const {isMobile} = useScreenSize();
+
+  useExpireMessage();
+
+  if (message == null) {
+    return null;
+  }
+
+  const closeIcon = (
+    <Close
+      size={24}
+      color={message.type === 'warning' ? color['gray-900'] : 'white'}
+      opacity={1}
+      className="z-20 hover:text-gray-300 color-white cursor-pointer !opacity-100 shrink-0"
+      onClick={clearMessage}
+    />
+  );
+
+  return (
+    <div
+      {...stylex.props(isMobile ? styles.mobileContainer : styles.container)}>
+      <div
+        {...stylex.props(
+          styles.messageContainer,
+          message.type === 'warning' && styles.messageWarningContainer,
+        )}>
+        <div {...stylex.props(styles.messageContent)}>
+          <div>{message.text}</div>
+          {message.type === 'loading' && <Loading size="xs" variant="dots" />}
+          {message.showClose && (
+            <div {...stylex.props(styles.closeColumn)}>
+              {message.expire ? (
+                <RadialProgress
+                  value={message.progress * 100}
+                  size="32px"
+                  thickness="2px"
+                  {...stylex.props(styles.progress)}>
+                  {closeIcon}
+                </RadialProgress>
+              ) : (
+                closeIcon
+              )}
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/snackbar/snackbarAtoms.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+
+export type MessageType = 'info' | 'loading' | 'warning';
+
+export type Message = {
+  type: MessageType;
+  text: string;
+  duration: number;
+  progress: number;
+  startTime: number;
+  expire: boolean;
+  showClose: boolean;
+  showReset: boolean;
+};
+
+export const messageAtom = atom<Message | null>(null);

demo/frontend/src/common/components/snackbar/useDemoMessagesSnackbar.ts

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {MessagesEventMap} from '@/common/components/snackbar/DemoMessagesSnackbarUtils';
+import useMessagesSnackbar from '@/common/components/snackbar/useMessagesSnackbar';
+import {messageMapAtom} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+import {useCallback} from 'react';
+
+type State = {
+  enqueueMessage: (messageType: keyof MessagesEventMap) => void;
+  clearMessage: () => void;
+};
+
+export default function useDemoMessagesSnackbar(): State {
+  const [messageMap, setMessageMap] = useAtom(messageMapAtom);
+  const {enqueueMessage: enqueue, clearMessage} = useMessagesSnackbar();
+
+  const enqueueMessage = useCallback(
+    (messageType: keyof MessagesEventMap) => {
+      const {text, shown, options} = messageMap[messageType];
+      if (!options?.repeat && shown === true) {
+        return;
+      }
+      enqueue(text, options);
+      const newState = {...messageMap};
+      newState[messageType].shown = true;
+      setMessageMap(newState);
+    },
+    [enqueue, messageMap, setMessageMap],
+  );
+
+  return {enqueueMessage, clearMessage};
+}

demo/frontend/src/common/components/snackbar/useExpireMessage.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useAtom} from 'jotai';
+import {useEffect, useRef} from 'react';
+import {Message, messageAtom} from '@/common/components/snackbar/snackbarAtoms';
+
+export default function useExpireMessage() {
+  const [message, setMessage] = useAtom(messageAtom);
+  const messageRef = useRef<Message | null>(null);
+  const intervalRef = useRef<NodeJS.Timeout | null>(null);
+
+  useEffect(() => {
+    messageRef.current = message;
+  }, [message]);
+
+  useEffect(() => {
+    function resetInterval() {
+      if (intervalRef.current != null) {
+        clearInterval(intervalRef.current);
+        intervalRef.current = null;
+      }
+    }
+    if (intervalRef.current == null && message != null && message.expire) {
+      intervalRef.current = setInterval(() => {
+        const prevMessage = messageRef.current;
+        if (prevMessage == null) {
+          setMessage(null);
+          resetInterval();
+          return;
+        }
+        const messageDuration = Date.now() - prevMessage.startTime;
+        if (messageDuration > prevMessage.duration) {
+          setMessage(null);
+          resetInterval();
+          return;
+        }
+        setMessage({
+          ...prevMessage,
+          progress: messageDuration / prevMessage.duration,
+        });
+      }, 20);
+    }
+  }, [message, setMessage]);
+
+  useEffect(() => {
+    return () => {
+      if (intervalRef.current != null) {
+        clearInterval(intervalRef.current);
+      }
+    };
+  }, []);
+}

demo/frontend/src/common/components/snackbar/useMessagesSnackbar.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useSetAtom} from 'jotai';
+import {useCallback} from 'react';
+import {
+  MessageType,
+  messageAtom,
+} from '@/common/components/snackbar/snackbarAtoms';
+
+export type EnqueueOption = {
+  duration?: number;
+  type?: MessageType;
+  expire?: boolean;
+  showClose?: boolean;
+  showReset?: boolean;
+};
+
+type State = {
+  clearMessage: () => void;
+  enqueueMessage: (message: string, options?: EnqueueOption) => void;
+};
+
+export default function useMessagesSnackbar(): State {
+  const setMessage = useSetAtom(messageAtom);
+
+  const enqueueMessage = useCallback(
+    (message: string, options?: EnqueueOption) => {
+      setMessage({
+        text: message,
+        type: options?.type ?? 'info',
+        duration: options?.duration ?? 5000,
+        progress: 0,
+        startTime: Date.now(),
+        expire: options?.expire ?? true,
+        showClose: options?.showClose ?? true,
+        showReset: options?.showReset ?? false,
+      });
+    },
+    [setMessage],
+  );
+
+  function clearMessage() {
+    setMessage(null);
+  }
+
+  return {enqueueMessage, clearMessage};
+}

demo/frontend/src/common/components/toolbar/DesktopToolbar.tsx

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ObjectsToolbar from '@/common/components/annotations/ObjectsToolbar';
+import EffectsToolbar from '@/common/components/effects/EffectsToolbar';
+import MoreOptionsToolbar from '@/common/components/options/MoreOptionsToolbar';
+import type {CSSProperties} from 'react';
+
+type Props = {
+  tabIndex: number;
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function DesktopToolbar({tabIndex, onTabChange}: Props) {
+  const toolbarShadow: CSSProperties = {
+    boxShadow: '0px 1px 3px 1px rgba(0,0,0,.25)',
+    transition: 'box-shadow 0.8s ease-out',
+  };
+
+  const tabs = [
+    <ObjectsToolbar key="objects" onTabChange={onTabChange} />,
+    <EffectsToolbar key="effects" onTabChange={onTabChange} />,
+    <MoreOptionsToolbar key="options" onTabChange={onTabChange} />,
+  ];
+
+  return (
+    <div
+      style={toolbarShadow}
+      className="bg-graydark-800 text-white md:basis-[350px] lg:basis-[435px] shrink-0 rounded-xl">
+      {tabs[tabIndex]}
+    </div>
+  );
+}

demo/frontend/src/common/components/toolbar/MobileToolbar.tsx

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import MobileObjectsToolbar from '@/common/components/annotations/MobileObjectsToolbar';
+import MobileEffectsToolbar from '@/common/components/effects/MobileEffectsToolbar';
+import MoreOptionsToolbar from '@/common/components/options/MoreOptionsToolbar';
+
+type Props = {
+  tabIndex: number;
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileToolbar({tabIndex, onTabChange}: Props) {
+  const tabs = [
+    <MobileObjectsToolbar key="objects" onTabChange={onTabChange} />,
+    <MobileEffectsToolbar key="effects" onTabChange={onTabChange} />,
+    <MoreOptionsToolbar key="more-options" onTabChange={onTabChange} />,
+  ];
+
+  return (
+    <div className="relative flex flex-col bg-black">{tabs[tabIndex]}</div>
+  );
+}

demo/frontend/src/common/components/toolbar/Toolbar.tsx

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useListenToStreamingState from '@/common/components/toolbar/useListenToStreamingState';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {
+  codeEditorOpenedAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+} from '@/demo/atoms';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+import DesktopToolbar from './DesktopToolbar';
+import MobileToolbar from './MobileToolbar';
+import {OBJECT_TOOLBAR_INDEX} from './ToolbarConfig';
+
+export default function Toolbar() {
+  const [tabIndex, setTabIndex] = useToolbarTabs();
+  const video = useVideo();
+  const setIsPlaying = useSetAtom(isPlayingAtom);
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const codeEditorOpened = useAtomValue(codeEditorOpenedAtom);
+  const {isMobile} = useScreenSize();
+  const setEffect = useVideoEffect();
+
+  const resetEffects = useCallback(() => {
+    setEffect('Original', EffectIndex.BACKGROUND, {variant: 0});
+    setEffect('Overlay', EffectIndex.HIGHLIGHT, {variant: 0});
+  }, [setEffect]);
+
+  const handleStopVideo = useCallback(() => {
+    if (isStreaming) {
+      video?.abortStreamMasks();
+    } else {
+      video?.pause();
+    }
+  }, [video, isStreaming]);
+
+  const handleTabChange = useCallback(
+    (newIndex: number) => {
+      if (newIndex === OBJECT_TOOLBAR_INDEX) {
+        handleStopVideo();
+        resetEffects();
+      }
+
+      setTabIndex(newIndex);
+    },
+    [handleStopVideo, resetEffects, setTabIndex],
+  );
+
+  useListenToStreamingState();
+
+  useEffect(() => {
+    function onPlay() {
+      setIsPlaying(true);
+    }
+    function onPause() {
+      setIsPlaying(false);
+    }
+
+    video?.addEventListener('play', onPlay);
+    video?.addEventListener('pause', onPause);
+    return () => {
+      video?.removeEventListener('play', onPlay);
+      video?.removeEventListener('pause', onPause);
+    };
+  }, [video, resetEffects, setIsStreaming, setIsPlaying]);
+
+  if (codeEditorOpened) {
+    return null;
+  }
+
+  return isMobile ? (
+    <MobileToolbar tabIndex={tabIndex} onTabChange={handleTabChange} />
+  ) : (
+    <DesktopToolbar tabIndex={tabIndex} onTabChange={handleTabChange} />
+  );
+}

demo/frontend/src/common/components/toolbar/ToolbarActionIcon.tsx

@@ -0,0 +1,99 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {Loading} from 'react-daisyui';
+
+type Props = {
+  isDisabled?: boolean;
+  isActive?: boolean;
+  icon: CarbonIconType;
+  title: string;
+  badge?: React.ReactNode;
+  variant: 'toggle' | 'button' | 'gradient' | 'flat';
+  span?: 1 | 2;
+  loadingProps?: {
+    loading: boolean;
+    label?: string;
+  };
+  onClick: () => void;
+};
+
+export default function ToolbarActionIcon({
+  variant,
+  isDisabled = false,
+  isActive = false,
+  title,
+  badge,
+  loadingProps,
+  icon: Icon,
+  span = 1,
+  onClick,
+}: Props) {
+  const {isMobile} = useScreenSize();
+  const isLoading = loadingProps?.loading === true;
+
+  function handleClick() {
+    if (isDisabled) {
+      return;
+    }
+    onClick();
+  }
+
+  const ButtonBase = (
+    <div
+      onClick={handleClick}
+      className={`relative rounded-lg h-full flex items-center justify-center select-none
+      ${!isDisabled && 'cursor-pointer hover:bg-black'}
+      ${span === 1 && 'col-span-1'} 
+      ${span === 2 && 'col-span-2'}
+      ${variant === 'button' && (isDisabled ? 'bg-graydark-500 text-gray-300' : 'bg-graydark-700 hover:bg-graydark-800 text-white')} 
+      ${variant === 'toggle' && (isActive ? BLUE_PINK_FILL_BR : 'bg-inherit')}
+      ${variant === 'flat' && (isDisabled ? ' text-gray-600' : 'text-white')} 
+      `}>
+      <div className="py-4 px-2">
+        <div className="flex items-center justify-center">
+          {isLoading ? (
+            <Loading size="md" className="mx-auto" />
+          ) : (
+            <Icon
+              size={isMobile ? 24 : 28}
+              color={isActive ? 'white' : 'black'}
+              className={`mx-auto ${isDisabled ? 'text-gray-300' : 'text-white'}`}
+            />
+          )}
+        </div>
+        <div
+          className={`mt-1 md:mt-2 text-center text-xs font-bold ${isActive && 'text-white'}`}>
+          {isLoading && loadingProps?.label != null
+            ? loadingProps.label
+            : title}
+        </div>
+        {isActive && badge}
+      </div>
+    </div>
+  );
+
+  return variant == 'gradient' ? (
+    <GradientBorder rounded={false} className="rounded-lg h-full text-white">
+      {ButtonBase}
+    </GradientBorder>
+  ) : (
+    ButtonBase
+  );
+}

demo/frontend/src/common/components/toolbar/ToolbarBottomActionsWrapper.tsx

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren} from 'react';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+    justifyContent: 'space-between',
+    paddingTop: {
+      default: spacing[2],
+      '@media screen and (max-width: 768px)': spacing[4],
+    },
+    paddingBottom: spacing[6],
+    paddingHorizontal: spacing[6],
+  },
+});
+
+export default function ToolbarBottomActionsWrapper({
+  children,
+}: PropsWithChildren) {
+  return <div {...stylex.props(styles.container)}>{children}</div>;
+}

demo/frontend/src/common/components/toolbar/ToolbarConfig.tsx

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export const OBJECT_TOOLBAR_INDEX = 0;
+export const EFFECT_TOOLBAR_INDEX = 1;
+export const MORE_OPTIONS_TOOLBAR_INDEX = 2;

demo/frontend/src/common/components/toolbar/ToolbarHeaderWrapper.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {ReactNode} from 'react';
+import ToolbarProgressChip from './ToolbarProgressChip';
+
+type Props = {
+  title: string;
+  description?: string;
+  bottomSection?: ReactNode;
+  showProgressChip?: boolean;
+  className?: string;
+};
+
+export default function ToolbarHeaderWrapper({
+  title,
+  description,
+  bottomSection,
+  showProgressChip = true,
+  className,
+}: Props) {
+  return (
+    <div
+      className={`flex flex-col gap-2 p-8 border-b border-b-black ${className}`}>
+      <div className="flex items-center">
+        {showProgressChip && <ToolbarProgressChip />}
+        <h2 className="text-xl">{title}</h2>
+      </div>
+
+      {description != null && (
+        <div className="flex-1 text-gray-400">{description}</div>
+      )}
+      {bottomSection != null && bottomSection}
+    </div>
+  );
+}

demo/frontend/src/common/components/toolbar/ToolbarProgressChip.tsx

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import {streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+import {useMemo} from 'react';
+import {Loading} from 'react-daisyui';
+
+const TOTAL_DEMO_STEPS = 3;
+
+export default function ToolbarProgressChip() {
+  const [toolbarIndex] = useToolbarTabs();
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  const showLoader = useMemo(() => {
+    return streamingState === 'partial' || streamingState === 'requesting';
+  }, [streamingState]);
+
+  function getStepValue() {
+    if (toolbarIndex === OBJECT_TOOLBAR_INDEX) {
+      return streamingState !== 'full' ? 1 : 2;
+    }
+    return 3;
+  }
+
+  return (
+    <span className="inline-flex items-center justify-center rounded-full text-xs md:text-sm font-medium bg-white text-black w-10 md:w-12 h-5 md:h-6 mr-2 shrink-0 ">
+      {showLoader ? (
+        <Loading className="w-2 md:w-4" />
+      ) : (
+        `${getStepValue()}/${TOTAL_DEMO_STEPS}`
+      )}
+    </span>
+  );
+}

demo/frontend/src/common/components/toolbar/ToolbarSection.tsx

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {PropsWithChildren} from 'react';
+
+type Props = PropsWithChildren<{
+  title: string;
+  borderBottom?: boolean;
+}>;
+
+export default function ToolbarSection({
+  children,
+  title,
+  borderBottom = false,
+}: Props) {
+  return (
+    <div className={`p-6 ${borderBottom && 'border-b border-black'}`}>
+      <div className="font-bold ml-2">{title}</div>
+      <div className="grid grid-cols-4 gap-2 mt-2 md:mt-6">{children}</div>
+    </div>
+  );
+}

demo/frontend/src/common/components/toolbar/useListenToStreamingState.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {StreamingStateUpdateEvent} from '@/common/components/video/VideoWorkerBridge';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {StreamingState} from '@/common/tracker/Tracker';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+import {useEffect} from 'react';
+
+export default function useListenToStreamingState(): {
+  isStreaming: boolean;
+  streamingState: StreamingState;
+} {
+  const [streamingState, setStreamingState] = useAtom(streamingStateAtom);
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const video = useVideo();
+
+  useEffect(() => {
+    function onStreamingStateUpdate(event: StreamingStateUpdateEvent) {
+      setStreamingState(event.state);
+    }
+    function onStreamingStarted() {
+      setIsStreaming(true);
+    }
+    function onStreamingCompleted() {
+      setIsStreaming(false);
+    }
+    video?.addEventListener('streamingStateUpdate', onStreamingStateUpdate);
+    video?.addEventListener('streamingStarted', onStreamingStarted);
+    video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+    return () => {
+      video?.removeEventListener(
+        'streamingStateUpdate',
+        onStreamingStateUpdate,
+      );
+      video?.removeEventListener('streamingStarted', onStreamingStarted);
+      video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+    };
+  }, [video, setStreamingState, setIsStreaming]);
+
+  return {isStreaming, streamingState};
+}

demo/frontend/src/common/components/toolbar/useToolbarTabs.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {toolbarTabIndex} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+
+type State = [tabIndex: number, setTabIndex: (tabIndex: number) => void];
+
+export default function useToolbarTabs(): State {
+  return useAtom(toolbarTabIndex);
+}

demo/frontend/src/common/components/useFunctionThrottle.tsx

@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useCallback, useState} from 'react';
+
+type ThrottleOptions = {
+  enableThrottling?: boolean;
+};
+type State = {
+  isThrottled: boolean;
+  maxThrottles: boolean;
+  throttle: (callback: () => void, options?: ThrottleOptions) => void;
+};
+
+export default function useFunctionThrottle(
+  initialDelay: number,
+  numThrottles: number,
+): State {
+  const [isThrottled, setIsThrottled] = useState<boolean>(false);
+  const [lastClickTime, setLastClickTime] = useState<number | null>(null);
+  const [numTimesThrottled, setNumTimesThrottled] = useState<number>(1);
+
+  /**
+   * The following function's callback gets throttled when the time between two
+   * executions is less than a threshold.
+   *
+   * The threshold is calculated linearly by multiplying the initial delay
+   * and the number of times the button has been throttled. The button can be
+   * throttled up to numThrottles times.
+   *
+   * The function has an optional flag - enableThrottling - which allows a callsite
+   * to optionally disable throttling. This is useful in cases where throttling may
+   * not be necessary. (e.g. for the Track & Play button, we would only like to
+   * throttle after a stream is aborted.)
+   */
+  const throttle = useCallback(
+    (
+      callback: () => void,
+      options: ThrottleOptions = {
+        enableThrottling: true,
+      },
+    ) => {
+      if (isThrottled) {
+        return;
+      }
+
+      const currentTime = Date.now();
+      if (lastClickTime == null) {
+        callback();
+        setLastClickTime(currentTime);
+        return;
+      }
+
+      const timeBetweenClicks = currentTime - lastClickTime;
+      const delay = initialDelay * numTimesThrottled;
+      const shouldThrottle =
+        options.enableThrottling && delay > timeBetweenClicks;
+
+      if (shouldThrottle) {
+        setIsThrottled(true);
+        setTimeout(() => {
+          setIsThrottled(false);
+        }, delay);
+        setNumTimesThrottled(prev => {
+          return prev === numThrottles ? numThrottles : prev + 1;
+        });
+      }
+
+      callback();
+      setLastClickTime(currentTime);
+    },
+    [initialDelay, numThrottles, isThrottled, lastClickTime, numTimesThrottled],
+  );
+
+  return {
+    isThrottled,
+    maxThrottles: numTimesThrottled === numThrottles,
+    throttle,
+  };
+}

demo/frontend/src/common/components/video/ChangeVideoModal.tsx

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import DemoVideoGalleryModal from '@/common/components/gallery/DemoVideoGalleryModal';
+import useVideo from '@/common/components/video/editor/useVideo';
+import Logger from '@/common/logger/Logger';
+import {isStreamingAtom, uploadingStateAtom, VideoData} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {ComponentType, useCallback} from 'react';
+import {useNavigate} from 'react-router-dom';
+
+type Props = {
+  videoGalleryModalTrigger?: ComponentType<VideoGalleryTriggerProps>;
+  showUploadInGallery?: boolean;
+  onChangeVideo?: () => void;
+};
+
+export default function ChangeVideoModal({
+  videoGalleryModalTrigger: VideoGalleryModalTriggerComponent,
+  showUploadInGallery = true,
+  onChangeVideo,
+}: Props) {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const video = useVideo();
+  const navigate = useNavigate();
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  function handlePauseOrAbortVideo() {
+    if (isStreaming) {
+      video?.abortStreamMasks();
+    } else {
+      handlePause();
+    }
+  }
+
+  function handleSwitchVideos(video: VideoData) {
+    // Retain any search parameter
+    navigate(
+      {
+        pathname: location.pathname,
+        search: location.search,
+      },
+      {
+        state: {
+          video,
+        },
+      },
+    );
+    onChangeVideo?.();
+  }
+
+  function handleUploadVideoError(error: Error) {
+    setUploadingState('error');
+    Logger.error(error);
+  }
+
+  return (
+    <DemoVideoGalleryModal
+      trigger={VideoGalleryModalTriggerComponent}
+      showUploadInGallery={showUploadInGallery}
+      onOpen={handlePauseOrAbortVideo}
+      onSelect={handleSwitchVideos}
+      onUploadVideoError={handleUploadVideoError}
+    />
+  );
+}

demo/frontend/src/common/components/video/EventEmitter.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type EventMap<WorkerEventMap> = {
+  type: keyof WorkerEventMap;
+  listener: (ev: WorkerEventMap[keyof WorkerEventMap]) => unknown;
+};
+
+export class EventEmitter<WorkerEventMap> {
+  listeners: EventMap<WorkerEventMap>[] = [];
+
+  trigger<K extends keyof WorkerEventMap>(type: K, ev: WorkerEventMap[K]) {
+    this.listeners
+      .filter(listener => type === listener.type)
+      .forEach(({listener}) => {
+        setTimeout(() => listener(ev), 0);
+      });
+  }
+
+  addEventListener<K extends keyof WorkerEventMap>(
+    type: K,
+    listener: (ev: WorkerEventMap[K]) => unknown,
+  ): void {
+    // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+    this.listeners.push({type, listener});
+  }
+
+  removeEventListener<K extends keyof WorkerEventMap>(
+    type: K,
+    listener: (ev: WorkerEventMap[K]) => unknown,
+  ): void {
+    this.listeners = this.listeners.filter(
+      existingListener =>
+        !(
+          existingListener.type === type &&
+          existingListener.listener === listener
+        ),
+    );
+  }
+
+  destroy() {
+    this.listeners.length = 0;
+  }
+}

demo/frontend/src/common/components/video/Video.tsx

@@ -0,0 +1,374 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BaseTracklet, SegmentationPoint} from '@/common/tracker/Tracker';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import stylex, {StyleXStyles} from '@stylexjs/stylex';
+import {
+  CSSProperties,
+  forwardRef,
+  useEffect,
+  useImperativeHandle,
+  useMemo,
+  useRef,
+} from 'react';
+import {Button} from 'react-daisyui';
+
+import {EffectIndex, Effects} from '@/common/components/video/effects/Effects';
+import useReportError from '@/common/error/useReportError';
+import Logger from '@/common/logger/Logger';
+import {isPlayingAtom, isVideoLoadingAtom} from '@/demo/atoms';
+import {color} from '@/theme/tokens.stylex';
+import {useAtom} from 'jotai';
+import useResizeObserver from 'use-resize-observer';
+import VideoLoadingOverlay from './VideoLoadingOverlay';
+import {
+  StreamingStateUpdateEvent,
+  VideoWorkerEventMap,
+} from './VideoWorkerBridge';
+import {EffectOptions} from './effects/Effect';
+import useVideoWorker from './useVideoWorker';
+
+const styles = stylex.create({
+  container: {
+    position: 'relative',
+    width: '100%',
+    height: '100%',
+  },
+  canvasContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    backgroundColor: color['gray-800'],
+    width: '100%',
+    height: '100%',
+  },
+  controls: {
+    position: 'absolute',
+    bottom: 0,
+    left: 0,
+    width: '100%',
+    padding: 8,
+    background: 'linear-gradient(#00000000, #000000ff)',
+  },
+  controlButton: {
+    color: 'white',
+  },
+});
+
+type Props = {
+  src: string;
+  width: number;
+  height: number;
+  loading?: boolean;
+  containerStyle?: StyleXStyles<{
+    position: CSSProperties['position'];
+  }>;
+  canvasStyle?: StyleXStyles<{
+    width: CSSProperties['width'];
+  }>;
+  controls?: boolean;
+  createVideoWorker?: () => Worker;
+};
+
+export type VideoRef = {
+  getCanvas(): HTMLCanvasElement | null;
+  get width(): number;
+  get height(): number;
+  get frame(): number;
+  set frame(index: number);
+  get numberOfFrames(): number;
+  play(): void;
+  pause(): void;
+  stop(): void;
+  previousFrame(): void;
+  nextFrame(): void;
+  setEffect(
+    name: keyof Effects,
+    index: EffectIndex,
+    options?: EffectOptions,
+  ): void;
+  encode(): void;
+  streamMasks(): void;
+  abortStreamMasks(): Promise<void>;
+  addEventListener<K extends keyof VideoWorkerEventMap>(
+    type: K,
+    listener: (ev: VideoWorkerEventMap[K]) => unknown,
+  ): void;
+  removeEventListener<K extends keyof VideoWorkerEventMap>(
+    type: K,
+    listener: (ev: VideoWorkerEventMap[K]) => unknown,
+  ): void;
+  createFilmstrip(width: number, height: number): Promise<ImageBitmap>;
+  // Tracker
+  initializeTracker(name: keyof Trackers, options?: TrackerOptions): void;
+  startSession(videoUrl: string): Promise<string | null>;
+  closeSession(): void;
+  logAnnotations(): void;
+  createTracklet(): Promise<BaseTracklet>;
+  deleteTracklet(trackletId: number): Promise<void>;
+  updatePoints(trackletId: number, points: SegmentationPoint[]): void;
+  clearPointsInVideo(): Promise<boolean>;
+  getWorker_ONLY_USE_WITH_CAUTION(): Worker;
+};
+
+export default forwardRef<VideoRef, Props>(function Video(
+  {
+    src,
+    width,
+    height,
+    containerStyle,
+    canvasStyle,
+    createVideoWorker,
+    controls = false,
+    loading = false,
+  },
+  ref,
+) {
+  const reportError = useReportError();
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const [isPlaying, setIsPlaying] = useAtom(isPlayingAtom);
+  const [isVideoLoading, setIsVideoLoading] = useAtom(isVideoLoadingAtom);
+
+  const bridge = useVideoWorker(src, canvasRef, {
+    createVideoWorker,
+  });
+
+  const {
+    ref: resizeObserverRef,
+    width: resizeWidth = 1,
+    height: resizeHeight = 1,
+  } = useResizeObserver<HTMLDivElement>();
+
+  const canvasHeight = useMemo(() => {
+    const resizeRatio = resizeWidth / width;
+    return Math.min(height * resizeRatio, resizeHeight);
+  }, [resizeWidth, height, width, resizeHeight]);
+
+  useImperativeHandle(
+    ref,
+    () => ({
+      getCanvas() {
+        return canvasRef.current;
+      },
+      get width() {
+        return bridge.width;
+      },
+      get height() {
+        return bridge.width;
+      },
+      get frame() {
+        return bridge.frame;
+      },
+      set frame(index: number) {
+        bridge.frame = index;
+      },
+      get numberOfFrames() {
+        return bridge.numberOfFrames;
+      },
+      play(): void {
+        bridge.play();
+      },
+      pause(): void {
+        bridge.pause();
+      },
+      stop(): void {
+        bridge.stop();
+      },
+      previousFrame(): void {
+        bridge.previousFrame();
+      },
+      nextFrame(): void {
+        bridge.nextFrame();
+      },
+      setEffect(
+        name: keyof Effects,
+        index: number,
+        options?: EffectOptions,
+      ): void {
+        bridge.setEffect(name, index, options);
+      },
+      encode(): void {
+        bridge.encode();
+      },
+      streamMasks(): void {
+        bridge.streamMasks();
+      },
+      abortStreamMasks(): Promise<void> {
+        return bridge.abortStreamMasks();
+      },
+      addEventListener<K extends keyof VideoWorkerEventMap>(
+        type: K,
+        listener: (ev: VideoWorkerEventMap[K]) => unknown,
+      ): void {
+        bridge.addEventListener(type, listener);
+      },
+      removeEventListener<K extends keyof VideoWorkerEventMap>(
+        type: K,
+        listener: (ev: VideoWorkerEventMap[K]) => unknown,
+      ): void {
+        bridge.removeEventListener(type, listener);
+      },
+      createFilmstrip(width: number, height: number): Promise<ImageBitmap> {
+        return bridge.createFilmstrip(width, height);
+      },
+      // Tracker
+      initializeTracker(name: keyof Trackers, options: TrackerOptions): void {
+        bridge.initializeTracker(name, options);
+      },
+      startSession(videoUrl: string): Promise<string | null> {
+        return bridge.startSession(videoUrl);
+      },
+      closeSession(): void {
+        bridge.closeSession();
+      },
+      logAnnotations(): void {
+        bridge.logAnnotations();
+      },
+      createTracklet(): Promise<BaseTracklet> {
+        return bridge.createTracklet();
+      },
+      deleteTracklet(trackletId: number): Promise<void> {
+        return bridge.deleteTracklet(trackletId);
+      },
+      updatePoints(trackletId: number, points: SegmentationPoint[]): void {
+        bridge.updatePoints(trackletId, points);
+      },
+      clearPointsInVideo(): Promise<boolean> {
+        return bridge.clearPointsInVideo();
+      },
+      getWorker_ONLY_USE_WITH_CAUTION() {
+        return bridge.getWorker_ONLY_USE_WITH_CAUTION();
+      },
+    }),
+    [bridge],
+  );
+
+  // Handle video playback events (get playback state to main thread)
+  useEffect(() => {
+    let isPlaying = false;
+
+    function onFocus() {
+      // Workaround for Safari where the video frame renders black on
+      // unknown events. Trigger re-render frame on focus.
+      if (!isPlaying) {
+        bridge.goToFrame(bridge.frame);
+      }
+    }
+
+    function onVisibilityChange() {
+      // Workaround for Safari where the video frame renders black on
+      // visibility change hidden. Returning to visible shows a black
+      // frame instead of rendering the current frame.
+      if (document.visibilityState === 'visible' && !isPlaying) {
+        bridge.goToFrame(bridge.frame);
+      }
+    }
+
+    function onError(event: ErrorEvent) {
+      const error = event.error;
+      Logger.error(error);
+      reportError(error);
+    }
+
+    function onPlay() {
+      isPlaying = true;
+      setIsPlaying(true);
+    }
+    function onPause() {
+      isPlaying = false;
+      setIsPlaying(false);
+    }
+
+    function onStreamingDone(event: StreamingStateUpdateEvent) {
+      // continue to play after streaming is done (state is "full")
+      if (event.state === 'full') {
+        bridge.play();
+      }
+    }
+
+    function onLoadStart() {
+      setIsVideoLoading(true);
+    }
+
+    function onDecodeStart() {
+      setIsVideoLoading(false);
+    }
+
+    window.addEventListener('focus', onFocus);
+    window.addEventListener('visibilitychange', onVisibilityChange);
+    bridge.addEventListener('error', onError);
+    bridge.addEventListener('play', onPlay);
+    bridge.addEventListener('pause', onPause);
+    bridge.addEventListener('streamingStateUpdate', onStreamingDone);
+    bridge.addEventListener('loadstart', onLoadStart);
+    bridge.addEventListener('decode', onDecodeStart);
+    return () => {
+      window.removeEventListener('focus', onFocus);
+      window.removeEventListener('visibilitychange', onVisibilityChange);
+      bridge.removeEventListener('error', onError);
+      bridge.removeEventListener('play', onPlay);
+      bridge.removeEventListener('pause', onPause);
+      bridge.removeEventListener('streamingStateUpdate', onStreamingDone);
+      bridge.removeEventListener('loadstart', onLoadStart);
+      bridge.removeEventListener('decode', onDecodeStart);
+    };
+  }, [bridge, reportError, setIsPlaying, setIsVideoLoading]);
+
+  return (
+    <div
+      {...stylex.props(containerStyle ?? styles.container)}
+      ref={resizeObserverRef}>
+      <div {...stylex.props(styles.canvasContainer)}>
+        {(isVideoLoading || loading) && <VideoLoadingOverlay />}
+        <canvas
+          ref={canvasRef}
+          {...stylex.props(canvasStyle)}
+          className="lg:rounded-[4px]"
+          width={width}
+          height={height}
+          style={{
+            height: canvasHeight,
+          }}
+        />
+      </div>
+      {controls && (
+        <div {...stylex.props(styles.controls)}>
+          <Button
+            color="ghost"
+            size="xs"
+            startIcon={
+              isPlaying ? (
+                <PauseFilled
+                  {...stylex.props(styles.controlButton)}
+                  size={14}
+                />
+              ) : (
+                <PlayFilledAlt
+                  {...stylex.props(styles.controlButton)}
+                  size={14}
+                />
+              )
+            }
+            onClick={() => {
+              isPlaying ? bridge.pause() : bridge.play();
+            }}
+          />
+        </div>
+      )}
+    </div>
+  );
+});

demo/frontend/src/common/components/video/VideoFilmstripWithPlayback.tsx

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PlaybackButton from '@/common/components/button/PlaybackButton';
+import VideoFilmstrip from '@/common/components/video/filmstrip/VideoFilmstrip';
+import {spacing, w} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'end',
+    gap: spacing[4],
+    paddingHorizontal: spacing[4],
+    width: '100%',
+  },
+  playbackButtonContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    width: w[12],
+    height: w[12],
+  },
+  filmstripContainer: {
+    flexGrow: 1,
+  },
+});
+
+export default function VideoFilmstripWithPlayback() {
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.playbackButtonContainer)}>
+        <PlaybackButton />
+      </div>
+      <div {...stylex.props(styles.filmstripContainer)}>
+        <VideoFilmstrip />
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/video/VideoLoadingOverlay.tsx

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {fontSize, fontWeight, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {Loading} from 'react-daisyui';
+
+const styles = stylex.create({
+  overlay: {
+    position: 'absolute',
+    width: '100%',
+    height: '100%',
+    background: 'rgba(0,0,0,0.5)',
+  },
+  indicatorContainer: {
+    position: 'absolute',
+    top: '50%',
+    left: '50%',
+    transform: 'translate(-50%, -50%)',
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[4],
+    color: 'white',
+  },
+  indicatorText: {
+    color: 'white',
+    fontSize: fontSize['sm'],
+    fontWeight: fontWeight['medium'],
+  },
+});
+
+type Props = {
+  label?: string;
+};
+
+export default function VideoLoadingOverlay({label}: Props) {
+  return (
+    <div {...stylex.props(styles.overlay)}>
+      <div {...stylex.props(styles.indicatorContainer)}>
+        <Loading size="sm" />
+        <div {...stylex.props(styles.indicatorText)}>
+          {label ?? 'Loading video...'}
+        </div>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/video/VideoWorker.ts

@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {Tracker} from '@/common/tracker/Tracker';
+import {TrackerRequestMessageEvent} from '@/common/tracker/TrackerTypes';
+import {TRACKER_MAPPING} from '@/common/tracker/Trackers';
+import {serializeError} from 'serialize-error';
+import VideoWorkerContext from './VideoWorkerContext';
+import {
+  ErrorResponse,
+  VideoWorkerRequestMessageEvent,
+} from './VideoWorkerTypes';
+
+registerSerializableConstructors();
+
+const context = new VideoWorkerContext();
+let tracker: Tracker | null = null;
+
+let statsEnabled = false;
+
+self.addEventListener(
+  'message',
+  async (
+    event: VideoWorkerRequestMessageEvent | TrackerRequestMessageEvent,
+  ) => {
+    try {
+      switch (event.data.action) {
+        // Initialize context
+        case 'setCanvas':
+          context.setCanvas(event.data.canvas);
+          break;
+        case 'setSource':
+          context.setSource(event.data.source);
+          break;
+
+        // Playback
+        case 'play':
+          context.play();
+          break;
+        case 'pause':
+          context.pause();
+          break;
+        case 'stop':
+          context.stop();
+          break;
+        case 'frameUpdate':
+          context.goToFrame(event.data.index);
+          break;
+
+        // Filmstrip
+        case 'filmstrip': {
+          const {width, height} = event.data;
+          await context.createFilmstrip(width, height);
+          break;
+        }
+
+        // Effects
+        case 'setEffect': {
+          const {name, index, options} = event.data;
+          await context.setEffect(name, index, options);
+          break;
+        }
+
+        // Encode
+        case 'encode': {
+          await context.encode();
+          break;
+        }
+
+        case 'enableStats': {
+          statsEnabled = true;
+          context.enableStats();
+          tracker?.enableStats();
+          break;
+        }
+
+        // Tracker
+        case 'initializeTracker': {
+          const {name, options} = event.data;
+          const Tracker = TRACKER_MAPPING[name];
+          // Update the endpoint for the streaming API
+          tracker = new Tracker(context, options);
+          if (statsEnabled) {
+            tracker.enableStats();
+          }
+          break;
+        }
+        case 'startSession': {
+          const {videoUrl} = event.data;
+          await tracker?.startSession(videoUrl);
+          break;
+        }
+        case 'createTracklet':
+          tracker?.createTracklet();
+          break;
+        case 'deleteTracklet':
+          await tracker?.deleteTracklet(event.data.trackletId);
+          break;
+        case 'closeSession':
+          tracker?.closeSession();
+          break;
+        case 'updatePoints': {
+          const {frameIndex, objectId, points} = event.data;
+          context.allowEffectAnimation(true, objectId, points);
+          await tracker?.updatePoints(frameIndex, objectId, points);
+          break;
+        }
+        case 'clearPointsInFrame': {
+          const {frameIndex, objectId} = event.data;
+          await tracker?.clearPointsInFrame(frameIndex, objectId);
+          break;
+        }
+        case 'clearPointsInVideo':
+          await tracker?.clearPointsInVideo();
+          break;
+        case 'streamMasks': {
+          const {frameIndex} = event.data;
+          context.allowEffectAnimation(false);
+          await tracker?.streamMasks(frameIndex);
+          break;
+        }
+        case 'abortStreamMasks':
+          tracker?.abortStreamMasks();
+          break;
+      }
+    } catch (error) {
+      const serializedError = serializeError(error);
+      const errorResponse: ErrorResponse = {
+        action: 'error',
+        error: serializedError,
+      };
+      self.postMessage(errorResponse);
+    }
+  },
+);

demo/frontend/src/common/components/video/VideoWorkerBridge.ts

@@ -0,0 +1,525 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {EffectIndex, Effects} from '@/common/components/video/effects/Effects';
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {
+  BaseTracklet,
+  SegmentationPoint,
+  StreamingState,
+} from '@/common/tracker/Tracker';
+import {
+  AbortStreamMasksRequest,
+  AddPointsResponse,
+  ClearPointsInFrameRequest,
+  ClearPointsInVideoRequest,
+  ClearPointsInVideoResponse,
+  CloseSessionRequest,
+  CreateTrackletRequest,
+  DeleteTrackletRequest,
+  InitializeTrackerRequest,
+  LogAnnotationsRequest,
+  SessionStartFailedResponse,
+  SessionStartedResponse,
+  StartSessionRequest,
+  StreamMasksRequest,
+  StreamingStateUpdateResponse,
+  TrackerRequest,
+  TrackerResponseMessageEvent,
+  TrackletCreatedResponse,
+  TrackletDeletedResponse,
+  UpdatePointsRequest,
+} from '@/common/tracker/TrackerTypes';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {MP4ArrayBuffer} from 'mp4box';
+import {deserializeError, type ErrorObject} from 'serialize-error';
+import {EventEmitter} from './EventEmitter';
+import {
+  EncodeVideoRequest,
+  FilmstripRequest,
+  FilmstripResponse,
+  FrameUpdateRequest,
+  PauseRequest,
+  PlayRequest,
+  SetCanvasRequest,
+  SetEffectRequest,
+  SetSourceRequest,
+  StopRequest,
+  VideoWorkerRequest,
+  VideoWorkerResponseMessageEvent,
+} from './VideoWorkerTypes';
+import {EffectOptions} from './effects/Effect';
+
+registerSerializableConstructors();
+
+export type DecodeEvent = {
+  totalFrames: number;
+  numFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+  done: boolean;
+};
+
+export type LoadStartEvent = unknown;
+
+export type EffectUpdateEvent = {
+  name: keyof Effects;
+  index: EffectIndex;
+  variant: number;
+  numVariants: number;
+};
+
+export type EncodingStateUpdateEvent = {
+  progress: number;
+};
+
+export type EncodingCompletedEvent = {
+  file: MP4ArrayBuffer;
+};
+
+export interface PlayEvent {}
+
+export interface PauseEvent {}
+
+export interface FilmstripEvent {
+  filmstrip: ImageBitmap;
+}
+
+export interface FrameUpdateEvent {
+  index: number;
+}
+
+export interface SessionStartedEvent {
+  sessionId: string;
+}
+
+export interface SessionStartFailedEvent {}
+
+export interface TrackletCreatedEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklet: BaseTracklet;
+}
+
+export interface TrackletsEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklets: BaseTracklet[];
+}
+
+export interface TrackletDeletedEvent {
+  isSuccessful: boolean;
+}
+
+export interface AddPointsEvent {
+  isSuccessful: boolean;
+}
+
+export interface ClearPointsInVideoEvent {
+  isSuccessful: boolean;
+}
+
+export interface StreamingStartedEvent {}
+
+export interface StreamingCompletedEvent {}
+
+export interface StreamingStateUpdateEvent {
+  state: StreamingState;
+}
+
+export interface RenderingErrorEvent {
+  error: ErrorObject;
+}
+
+export interface VideoWorkerEventMap {
+  error: ErrorEvent;
+  decode: DecodeEvent;
+  encodingStateUpdate: EncodingStateUpdateEvent;
+  encodingCompleted: EncodingCompletedEvent;
+  play: PlayEvent;
+  pause: PauseEvent;
+  filmstrip: FilmstripEvent;
+  frameUpdate: FrameUpdateEvent;
+  sessionStarted: SessionStartedEvent;
+  sessionStartFailed: SessionStartFailedEvent;
+  trackletCreated: TrackletCreatedEvent;
+  trackletsUpdated: TrackletsEvent;
+  trackletDeleted: TrackletDeletedEvent;
+  addPoints: AddPointsEvent;
+  clearPointsInVideo: ClearPointsInVideoEvent;
+  streamingStarted: StreamingStartedEvent;
+  streamingCompleted: StreamingCompletedEvent;
+  streamingStateUpdate: StreamingStateUpdateEvent;
+  // HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+  loadstart: LoadStartEvent;
+  effectUpdate: EffectUpdateEvent;
+  renderingError: RenderingErrorEvent;
+}
+
+type Metadata = {
+  totalFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+};
+
+export default class VideoWorkerBridge extends EventEmitter<VideoWorkerEventMap> {
+  static create(workerFactory: () => Worker) {
+    const worker = workerFactory();
+    return new VideoWorkerBridge(worker);
+  }
+
+  protected worker: Worker;
+  private metadata: Metadata | null = null;
+  private frameIndex: number = 0;
+
+  private _sessionId: string | null = null;
+
+  public get sessionId() {
+    return this._sessionId;
+  }
+
+  public get width() {
+    return this.metadata?.width ?? 0;
+  }
+
+  public get height() {
+    return this.metadata?.height ?? 0;
+  }
+
+  public get numberOfFrames() {
+    return this.metadata?.totalFrames ?? 0;
+  }
+
+  public get fps() {
+    return this.metadata?.fps ?? 0;
+  }
+
+  public get frame() {
+    return this.frameIndex;
+  }
+
+  constructor(worker: Worker) {
+    super();
+    this.worker = worker;
+
+    worker.addEventListener(
+      'message',
+      (
+        event: VideoWorkerResponseMessageEvent | TrackerResponseMessageEvent,
+      ) => {
+        switch (event.data.action) {
+          case 'error':
+            // Deserialize error before triggering the event
+            event.data.error = deserializeError(event.data.error);
+            break;
+          case 'decode':
+            this.metadata = event.data;
+            break;
+          case 'frameUpdate':
+            this.frameIndex = event.data.index;
+            break;
+          case 'sessionStarted':
+            this._sessionId = event.data.sessionId;
+            break;
+        }
+        this.trigger(event.data.action, event.data);
+      },
+    );
+  }
+
+  public setCanvas(canvas: HTMLCanvasElement): void {
+    const offscreenCanvas = canvas.transferControlToOffscreen();
+    this.sendRequest<SetCanvasRequest>(
+      'setCanvas',
+      {
+        canvas: offscreenCanvas,
+      },
+      [offscreenCanvas],
+    );
+  }
+
+  public setSource(source: string): void {
+    this.sendRequest<SetSourceRequest>('setSource', {
+      source,
+    });
+  }
+
+  public terminate(): void {
+    super.destroy();
+    this.worker.terminate();
+  }
+
+  public play(): void {
+    this.sendRequest<PlayRequest>('play');
+  }
+
+  public pause(): void {
+    this.sendRequest<PauseRequest>('pause');
+  }
+
+  public stop(): void {
+    this.sendRequest<StopRequest>('stop');
+  }
+
+  public goToFrame(index: number): void {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {
+      index,
+    });
+  }
+
+  public previousFrame(): void {
+    const index = Math.max(0, this.frameIndex - 1);
+    this.goToFrame(index);
+  }
+
+  public nextFrame(): void {
+    const index = Math.min(this.frameIndex + 1, this.numberOfFrames - 1);
+    this.goToFrame(index);
+  }
+
+  public set frame(index: number) {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {index});
+  }
+
+  createFilmstrip(width: number, height: number): Promise<ImageBitmap> {
+    return new Promise((resolve, _reject) => {
+      const handleFilmstripResponse = (
+        event: MessageEvent<FilmstripResponse>,
+      ) => {
+        if (event.data.action === 'filmstrip') {
+          this.worker.removeEventListener('message', handleFilmstripResponse);
+          resolve(event.data.filmstrip);
+        }
+      };
+
+      this.worker.addEventListener('message', handleFilmstripResponse);
+
+      this.sendRequest<FilmstripRequest>('filmstrip', {
+        width,
+        height,
+      });
+    });
+  }
+
+  setEffect(name: keyof Effects, index: EffectIndex, options?: EffectOptions) {
+    this.sendRequest<SetEffectRequest>('setEffect', {
+      name,
+      index,
+      options,
+    });
+  }
+
+  encode(): void {
+    this.sendRequest<EncodeVideoRequest>('encode');
+  }
+
+  initializeTracker(name: keyof Trackers, options: TrackerOptions): void {
+    this.sendRequest<InitializeTrackerRequest>('initializeTracker', {
+      name,
+      options,
+    });
+  }
+
+  startSession(videoUrl: string): Promise<string | null> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<
+          SessionStartedResponse | SessionStartFailedResponse
+        >,
+      ) => {
+        if (event.data.action === 'sessionStarted') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.sessionId);
+        }
+        if (event.data.action === 'sessionStartFailed') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(null);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<StartSessionRequest>('startSession', {
+        videoUrl,
+      });
+    });
+  }
+
+  closeSession(): void {
+    this.sendRequest<CloseSessionRequest>('closeSession');
+  }
+
+  logAnnotations(): void {
+    this.sendRequest<LogAnnotationsRequest>('logAnnotations');
+  }
+
+  createTracklet(): Promise<BaseTracklet> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<TrackletCreatedResponse>) => {
+        if (event.data.action === 'trackletCreated') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.tracklet);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<CreateTrackletRequest>('createTracklet');
+    });
+  }
+
+  deleteTracklet(trackletId: number): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const handleResponse = (event: MessageEvent<TrackletDeletedResponse>) => {
+        if (event.data.action === 'trackletDeleted') {
+          this.worker.removeEventListener('message', handleResponse);
+          if (event.data.isSuccessful) {
+            resolve();
+          } else {
+            reject(`could not delete tracklet ${trackletId}`);
+          }
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<DeleteTrackletRequest>('deleteTracklet', {trackletId});
+    });
+  }
+
+  updatePoints(
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<AddPointsResponse>) => {
+        if (event.data.action === 'addPoints') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<UpdatePointsRequest>('updatePoints', {
+        frameIndex: this.frame,
+        objectId,
+        points,
+      });
+    });
+  }
+
+  clearPointsInFrame(objectId: number) {
+    this.sendRequest<ClearPointsInFrameRequest>('clearPointsInFrame', {
+      frameIndex: this.frame,
+      objectId,
+    });
+  }
+
+  clearPointsInVideo(): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<ClearPointsInVideoResponse>,
+      ) => {
+        if (event.data.action === 'clearPointsInVideo') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<ClearPointsInVideoRequest>('clearPointsInVideo');
+    });
+  }
+
+  streamMasks(): void {
+    this.sendRequest<StreamMasksRequest>('streamMasks', {
+      frameIndex: this.frame,
+    });
+  }
+
+  abortStreamMasks(): Promise<void> {
+    return new Promise(resolve => {
+      const handleAbortResponse = (
+        event: MessageEvent<StreamingStateUpdateResponse>,
+      ) => {
+        if (
+          event.data.action === 'streamingStateUpdate' &&
+          event.data.state === 'aborted'
+        ) {
+          this.worker.removeEventListener('message', handleAbortResponse);
+          resolve();
+        }
+      };
+
+      this.worker.addEventListener('message', handleAbortResponse);
+      this.sendRequest<AbortStreamMasksRequest>('abortStreamMasks');
+    });
+  }
+
+  getWorker_ONLY_USE_WITH_CAUTION(): Worker {
+    return this.worker;
+  }
+
+  /**
+   * Convenient function to have typed postMessage.
+   *
+   * @param action Video worker action
+   * @param message Actual payload
+   * @param transfer Any object that should be transferred instead of cloned
+   */
+  protected sendRequest<T extends VideoWorkerRequest | TrackerRequest>(
+    action: T['action'],
+    payload?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ) {
+    this.worker.postMessage(
+      {
+        action,
+        ...payload,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  // // Override EventEmitter
+
+  // addEventListener<K extends keyof WorkerEventMap>(
+  //   type: K,
+  //   listener: (ev: WorkerEventMap[K]) => unknown,
+  // ): void {
+  //   switch (type) {
+  //     case 'frameUpdate':
+  //       {
+  //         const event: FrameUpdateEvent = {
+  //           index: this.frameIndex,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //       break;
+  //     case 'sessionStarted': {
+  //       if (this.sessionId !== null) {
+  //         const event: SessionStartedEvent = {
+  //           sessionId: this.sessionId,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //     }
+  //   }
+  //   super.addEventListener(type, listener);
+  // }
+}

demo/frontend/src/common/components/video/VideoWorkerContext.ts

@@ -0,0 +1,843 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodedVideo,
+  ImageFrame,
+  decodeStream,
+} from '@/common/codecs/VideoDecoder';
+import {encode as encodeVideo} from '@/common/codecs/VideoEncoder';
+import {
+  Effect,
+  EffectActionPoint,
+  EffectFrameContext,
+  EffectOptions,
+} from '@/common/components/video/effects/Effect';
+import AllEffects, {
+  EffectIndex,
+  Effects,
+} from '@/common/components/video/effects/Effects';
+import Logger from '@/common/logger/Logger';
+import {Mask, SegmentationPoint, Tracklet} from '@/common/tracker/Tracker';
+import {streamFile} from '@/common/utils/FileUtils';
+import {Stats} from '@/debug/stats/Stats';
+import {VIDEO_WATERMARK_TEXT} from '@/demo/DemoConfig';
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {RLEObject} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+import {serializeError} from 'serialize-error';
+import {
+  DecodeResponse,
+  EffectUpdateResponse,
+  EncodingCompletedResponse,
+  EncodingStateUpdateResponse,
+  FilmstripResponse,
+  FrameUpdateResponse,
+  PauseRequest,
+  PlayRequest,
+  RenderingErrorResponse,
+  VideoWorkerResponse,
+} from './VideoWorkerTypes';
+
+function getEvenlySpacedItems(decodedVideo: DecodedVideo, x: number) {
+  const p = Math.floor(decodedVideo.numFrames / Math.max(1, x - 1));
+  const middleFrames = decodedVideo.frames
+    .slice(p, p * x)
+    .filter(function (_, i) {
+      return 0 == i % p;
+    });
+  return [
+    decodedVideo.frames[0],
+    ...middleFrames,
+    decodedVideo.frames[decodedVideo.numFrames - 1],
+  ];
+}
+
+export type FrameInfo = {
+  tracklet: Tracklet;
+  mask: Mask;
+};
+
+const WATERMARK_BOX_HORIZONTAL_PADDING = 10;
+const WATERMARK_BOX_VERTICAL_PADDING = 10;
+
+export type VideoStats = {
+  fps?: Stats;
+  videoFps?: Stats;
+  total?: Stats;
+  effect0?: Stats;
+  effect1?: Stats;
+  frameBmp?: Stats;
+  maskBmp?: Stats;
+  memory?: Stats;
+};
+
+export default class VideoWorkerContext {
+  private _canvas: OffscreenCanvas | null = null;
+  private _stats: VideoStats = {};
+  private _ctx: OffscreenCanvasRenderingContext2D | null = null;
+  private _form: CanvasForm | null = null;
+  private _decodedVideo: DecodedVideo | null = null;
+  private _frameIndex: number = 0;
+  private _isPlaying: boolean = false;
+  private _playbackRAFHandle: number | null = null;
+  private _playbackTimeoutHandle: NodeJS.Timeout | null = null;
+  private _isDrawing: boolean = false;
+  private _glObjects: WebGL2RenderingContext | null = null;
+  private _glBackground: WebGL2RenderingContext | null = null;
+  private _canvasHighlights: OffscreenCanvas | null = null;
+  private _canvasBackground: OffscreenCanvas | null = null;
+  private _allowAnimation: boolean = false;
+  private _currentSegmetationPoint: EffectActionPoint | null = null;
+
+  private _effects: Effect[];
+  private _tracklets: Tracklet[] = [];
+
+  public get width(): number {
+    return this._decodedVideo?.width ?? 0;
+  }
+
+  public get height(): number {
+    return this._decodedVideo?.height ?? 0;
+  }
+
+  public get frameIndex(): number {
+    return this._frameIndex;
+  }
+
+  public get currentFrame(): VideoFrame | null {
+    return this._decodedVideo?.frames[this._frameIndex].bitmap ?? null;
+  }
+
+  constructor() {
+    this._effects = [
+      AllEffects.Original, // Image as background
+      AllEffects.Overlay, // Masks on top
+    ];
+
+    // Loading watermark fonts. This is going to be async, but by the time of
+    // video encoding, the fonts should be available.
+    this._loadWatermarkFonts();
+  }
+
+  private initializeWebGLContext(width: number, height: number): void {
+    // Given that we use highlight and background effects as layers,
+    // we need to create two WebGL contexts, one for each set.
+    // To avoid memory leaks and too many active contexts,
+    // these contexts must be re-used over the lifecycle of the session.
+
+    if (this._canvasHighlights == null && this._glObjects == null) {
+      this._canvasHighlights = new OffscreenCanvas(width, height);
+      this._glObjects = this._canvasHighlights.getContext('webgl2');
+
+      this._canvasHighlights.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasHighlights != null &&
+      (this._canvasHighlights.width !== width ||
+        this._canvasHighlights.height !== height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasHighlights.width = width;
+      this._canvasHighlights.height = height;
+      if (this._glObjects != null) {
+        this._glObjects.viewport(0, 0, width, height);
+      }
+    }
+
+    if (this._canvasBackground == null && this._glBackground == null) {
+      this._canvasBackground = new OffscreenCanvas(width, height);
+      this._glBackground = this._canvasBackground.getContext('webgl2');
+
+      this._canvasBackground.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasBackground != null &&
+      (this._canvasBackground.width != width ||
+        this._canvasBackground.height != height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasBackground.width = width;
+      this._canvasBackground.height = height;
+      if (this._glBackground != null) {
+        this._glBackground.viewport(0, 0, width, height);
+      }
+    }
+  }
+
+  public setCanvas(canvas: OffscreenCanvas) {
+    this._canvas = canvas;
+    this._ctx = canvas.getContext('2d');
+    if (this._ctx == null) {
+      throw new Error('could not initialize drawing context');
+    }
+    this._form = new CanvasForm(this._ctx);
+  }
+
+  public setSource(src: string) {
+    this.close();
+
+    // Clear state of previous source.
+    this.updateFrameIndex(0);
+    this._tracklets = [];
+
+    this._decodeVideo(src);
+  }
+
+  public goToFrame(index: number): void {
+    // Cancel any ongoing render
+    this._cancelRender();
+    this.updateFrameIndex(index);
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  public play(): void {
+    // Video already playing
+    if (this._isPlaying) {
+      return;
+    }
+
+    // Cannot playback without frames
+    if (this._decodedVideo === null) {
+      throw new Error('no decoded video');
+    }
+
+    const {numFrames, fps} = this._decodedVideo;
+    const timePerFrame = 1000 / (fps ?? 30);
+    let startTime: number | null = null;
+    // The offset frame index compensate for cases where the video playback
+    // does not start at frame index 0.
+    const offsetFrameIndex = this._frameIndex;
+
+    const updateFrame = (time: number) => {
+      if (startTime === null) {
+        startTime = time;
+      }
+
+      this._stats.fps?.begin();
+
+      const diff = time - startTime;
+      const expectedFrame =
+        (Math.floor(diff / timePerFrame) + offsetFrameIndex) % numFrames;
+
+      if (this._frameIndex !== expectedFrame && !this._isDrawing) {
+        // Update to the next expected frame
+        this.updateFrameIndex(expectedFrame);
+        this._drawFrame();
+      }
+
+      this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+
+      this._stats.fps?.end();
+    };
+
+    this.updatePlayback(true);
+    this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+  }
+
+  public pause(): void {
+    this.updatePlayback(false);
+    this._cancelRender();
+  }
+
+  public stop(): void {
+    this.pause();
+    this.updateFrameIndex(0);
+  }
+
+  public async createFilmstrip(width: number, height: number): Promise<void> {
+    if (width < 1 || height < 1) {
+      Logger.warn(
+        `Cannot create filmstrip because width ${width} or height ${height} is too small.`,
+      );
+      return;
+    }
+
+    try {
+      const canvas = new OffscreenCanvas(width, height);
+      const ctx = canvas.getContext('2d');
+
+      if (this._decodedVideo !== null) {
+        const scale = canvas.height / this._decodedVideo.height;
+        const resizeWidth = this._decodedVideo.width * scale;
+
+        const spacedFrames = getEvenlySpacedItems(
+          this._decodedVideo,
+          Math.ceil(canvas.width / resizeWidth),
+        );
+
+        spacedFrames.forEach((frame, idx) => {
+          if (frame != null) {
+            ctx?.drawImage(
+              frame.bitmap,
+              resizeWidth * idx,
+              0,
+              resizeWidth,
+              canvas.height,
+            );
+          }
+        });
+      }
+
+      const filmstrip = await createImageBitmap(canvas);
+
+      this.sendResponse<FilmstripResponse>(
+        'filmstrip',
+        {
+          filmstrip,
+        },
+        [filmstrip],
+      );
+    } catch {
+      this._sendRenderingError(
+        new CreateFilmstripError('Failed to create filmstrip'),
+      );
+    }
+  }
+
+  public async setEffect(
+    name: keyof Effects,
+    index: EffectIndex,
+    options?: EffectOptions,
+  ): Promise<void> {
+    const effect: Effect = AllEffects[name];
+
+    // The effect has changed.
+    if (this._effects[index] !== effect) {
+      // Effect changed. Cleanup old effect first. Effects are responsible for
+      // cleaning up their memory.
+      await this._effects[index].cleanup();
+
+      const offCanvas =
+        index === EffectIndex.BACKGROUND
+          ? this._canvasBackground
+          : this._canvasHighlights;
+      invariant(offCanvas != null, 'need OffscreenCanvas to render effects');
+      const webglContext =
+        index === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+      invariant(webglContext != null, 'need WebGL context to render effects');
+
+      // Initialize the effect. This can be used by effects to prepare
+      // resources needed for rendering. If the video wasn't decoded yet, the
+      // effect setup will happen in the _decodeVideo function.
+      if (this._decodedVideo != null) {
+        await effect.setup({
+          width: this._decodedVideo.width,
+          height: this._decodedVideo.height,
+          canvas: offCanvas,
+          gl: webglContext,
+        });
+      }
+    }
+
+    // Update effect if already set effect was clicked again. This can happen
+    // when there is a new variant of the effect.
+    if (options != null) {
+      // Update effect if already set effect was clicked again. This can happen
+      // when there is a new variant of the effect.
+      await effect.update(options);
+    }
+
+    // Notify the frontend about the effect state including its variant.
+    this.sendResponse<EffectUpdateResponse>('effectUpdate', {
+      name,
+      index,
+      variant: effect.variant,
+      numVariants: effect.numVariants,
+    });
+
+    this._effects[index] = effect;
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  async encode() {
+    const decodedVideo = this._decodedVideo;
+    invariant(
+      decodedVideo !== null,
+      'cannot encode video because there is no decoded video available',
+    );
+
+    const canvas = new OffscreenCanvas(this.width, this.height);
+    const ctx = canvas.getContext('2d', {willReadFrequently: true});
+    invariant(
+      ctx !== null,
+      'cannot encode video because failed to construct offscreen canvas context',
+    );
+
+    const form = new CanvasForm(ctx);
+
+    const file = await encodeVideo(
+      this.width,
+      this.height,
+      decodedVideo.frames.length,
+      this._framesGenerator(decodedVideo, canvas, form),
+      progress => {
+        this.sendResponse<EncodingStateUpdateResponse>('encodingStateUpdate', {
+          progress,
+        });
+      },
+    );
+    this.sendResponse<EncodingCompletedResponse>(
+      'encodingCompleted',
+      {
+        file,
+      },
+      [file],
+    );
+  }
+
+  private async *_framesGenerator(
+    decodedVideo: DecodedVideo,
+    canvas: OffscreenCanvas,
+    form: CanvasForm,
+  ): AsyncGenerator<ImageFrame, undefined> {
+    const frames = decodedVideo.frames;
+
+    for (let frameIndex = 0; frameIndex < frames.length; ++frameIndex) {
+      await this._drawFrameImpl(form, frameIndex, true);
+
+      const frame = frames[frameIndex];
+      const videoFrame = new VideoFrame(canvas, {
+        timestamp: frame.bitmap.timestamp,
+      });
+
+      yield {
+        bitmap: videoFrame,
+        timestamp: frame.timestamp,
+        duration: frame.duration,
+      };
+
+      videoFrame.close();
+    }
+  }
+
+  public enableStats() {
+    this._stats.fps = new Stats('fps');
+    this._stats.videoFps = new Stats('fps', 'V');
+    this._stats.total = new Stats('ms', 'T');
+    this._stats.effect0 = new Stats('ms', 'B');
+    this._stats.effect1 = new Stats('ms', 'H');
+    this._stats.frameBmp = new Stats('ms', 'F');
+    this._stats.maskBmp = new Stats('ms', 'M');
+    this._stats.memory = new Stats('memory');
+  }
+
+  public allowEffectAnimation(
+    allow: boolean = true,
+    objectId?: number,
+    points?: SegmentationPoint[],
+  ) {
+    if (objectId != null && points != null && points.length) {
+      const last_point_position = points[points.length - 1];
+      this._currentSegmetationPoint = {
+        objectId,
+        position: [last_point_position[0], last_point_position[1]],
+      };
+    }
+
+    if (!allow) {
+      this._currentSegmetationPoint = null;
+    }
+
+    this._allowAnimation = allow;
+  }
+
+  public close(): void {
+    // Clear any frame content
+    this._ctx?.reset();
+
+    // Close frames of previously decoded video.
+    this._decodedVideo?.frames.forEach(f => f.bitmap.close());
+    this._decodedVideo = null;
+  }
+
+  // TRACKER
+
+  public updateTracklets(
+    frameIndex: number,
+    tracklets: Tracklet[],
+    shouldGoToFrame: boolean = true,
+  ): void {
+    this._tracklets = tracklets;
+    if (shouldGoToFrame) {
+      this.goToFrame(frameIndex);
+    }
+  }
+
+  public clearTrackletMasks(tracklet: Tracklet): void {
+    this._tracklets = this._tracklets.filter(t => t.id != tracklet.id);
+  }
+
+  public clearMasks(): void {
+    this._tracklets = [];
+  }
+
+  // PRIVATE FUNCTIONS
+
+  private sendResponse<T extends VideoWorkerResponse>(
+    action: T['action'],
+    message?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ): void {
+    self.postMessage(
+      {
+        action,
+        ...message,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  private async _decodeVideo(src: string): Promise<void> {
+    const canvas = this._canvas;
+    invariant(canvas != null, 'need canvas to render decoded video');
+
+    this.sendResponse('loadstart');
+
+    const fileStream = streamFile(src, {
+      credentials: 'same-origin',
+      cache: 'no-store',
+    });
+
+    let renderedFirstFrame = false;
+    this._decodedVideo = await decodeStream(fileStream, async progress => {
+      const {fps, height, width, numFrames, frames} = progress;
+      this._decodedVideo = progress;
+      if (!renderedFirstFrame) {
+        renderedFirstFrame = true;
+        canvas.width = width;
+        canvas.height = height;
+        // Set WebGL contexts right after the first frame decoded
+        this.initializeWebGLContext(width, height);
+
+        // Initialize effect once first frame was decoded.
+        for (const [i, effect] of this._effects.entries()) {
+          const offCanvas =
+            i === EffectIndex.BACKGROUND
+              ? this._canvasBackground
+              : this._canvasHighlights;
+          invariant(offCanvas != null, 'need canvas to render effects');
+          const webglContext =
+            i === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+          invariant(
+            webglContext != null,
+            'need WebGL context to render effects',
+          );
+          await effect.setup({
+            width,
+            height,
+            canvas: offCanvas,
+            gl: webglContext,
+          });
+        }
+
+        // Need to render frame immediately. Cannot go through
+        // requestAnimationFrame because then rendering this frame would be
+        // delayed until the full video has finished decoding.
+        this._drawFrame();
+
+        this._stats.videoFps?.updateMaxValue(fps);
+        this._stats.total?.updateMaxValue(1000 / fps);
+        this._stats.effect0?.updateMaxValue(1000 / fps);
+        this._stats.effect1?.updateMaxValue(1000 / fps);
+        this._stats.frameBmp?.updateMaxValue(1000 / fps);
+        this._stats.maskBmp?.updateMaxValue(1000 / fps);
+      }
+      this.sendResponse<DecodeResponse>('decode', {
+        totalFrames: numFrames,
+        numFrames: frames.length,
+        fps: fps,
+        width: width,
+        height: height,
+        done: false,
+      });
+    });
+
+    if (!renderedFirstFrame) {
+      canvas.width = this._decodedVideo.width;
+      canvas.height = this._decodedVideo.height;
+      this._drawFrame();
+    }
+
+    this.sendResponse<DecodeResponse>('decode', {
+      totalFrames: this._decodedVideo.numFrames,
+      numFrames: this._decodedVideo.frames.length,
+      fps: this._decodedVideo.fps,
+      width: this._decodedVideo.width,
+      height: this._decodedVideo.height,
+      done: true,
+    });
+  }
+
+  private _drawFrame(): void {
+    if (this._canvas !== null && this._form !== null) {
+      this._drawFrameImpl(this._form, this._frameIndex);
+    }
+  }
+
+  private async _drawFrameImpl(
+    form: CanvasForm,
+    frameIndex: number,
+    enableWatermark: boolean = false,
+    step: number = 0,
+    maxSteps: number = 40,
+  ): Promise<void> {
+    if (this._decodedVideo === null) {
+      return;
+    }
+
+    {
+      this._stats.videoFps?.begin();
+      this._stats.total?.begin();
+      this._stats.memory?.begin();
+    }
+
+    try {
+      const frame = this._decodedVideo.frames[frameIndex];
+      const {bitmap} = frame;
+
+      this._stats.frameBmp?.begin();
+
+      // Need to convert VideoFrame to ImageBitmap because Safari can only apply
+      // globalCompositeOperation on ImageBitmap and fails on VideoFrame. FWIW,
+      // Chrome treats VideoFrame similarly to ImageBitmap.
+      const frameBitmap = await createImageBitmap(bitmap);
+
+      this._stats.frameBmp?.end();
+
+      const masks: Mask[] = [];
+      const colors: string[] = [];
+      const tracklets: Tracklet[] = [];
+      this._tracklets.forEach(tracklet => {
+        const mask = tracklet.masks[frameIndex];
+        if (mask != null) {
+          masks.push(mask);
+          tracklets.push(tracklet);
+          colors.push(tracklet.color);
+        }
+      });
+      const effectActionPoint = this._currentSegmetationPoint;
+
+      this._stats.maskBmp?.begin();
+
+      const effectMaskPromises = masks.map(async ({data, bounds}) => {
+        return {
+          bounds,
+          bitmap: data as RLEObject,
+        };
+      });
+      const effectMasks = await Promise.all(effectMaskPromises);
+
+      this._stats.maskBmp?.end();
+
+      form.ctx.fillStyle = 'rgba(0, 0, 0, 0)';
+      form.ctx.fillRect(0, 0, this.width, this.height);
+
+      const effectParams: EffectFrameContext = {
+        frame: frameBitmap,
+        masks: effectMasks,
+        maskColors: colors,
+        frameIndex: frameIndex,
+        totalFrames: this._decodedVideo.frames.length,
+        fps: this._decodedVideo.fps,
+        width: frameBitmap.width,
+        height: frameBitmap.height,
+        actionPoint: null,
+      };
+
+      // Allows animation within a single frame.
+      if (this._allowAnimation && step < maxSteps) {
+        const animationDuration = 2; // Total duration of the animation in seconds
+        const progress = step / maxSteps;
+        const timeParameter = progress * animationDuration;
+        // Pass dynamic effect params
+        effectParams.timeParameter = timeParameter;
+        effectParams.actionPoint = effectActionPoint;
+
+        this._processEffects(form, effectParams, tracklets);
+
+        // Use RAF to draw frame, and update the display,
+        // this avoids to wait until the javascript call stack is cleared.
+        requestAnimationFrame(() =>
+          this._drawFrameImpl(form, frameIndex, false, step + 1, maxSteps),
+        );
+      } else {
+        this._processEffects(form, effectParams, tracklets);
+      }
+
+      if (enableWatermark) {
+        this._drawWatermark(form, frameBitmap);
+      }
+
+      // Do not simply drop the JavaScript reference to the ImageBitmap; doing so
+      // will keep its graphics resource alive until the next time the garbage
+      // collector runs.
+      frameBitmap.close();
+
+      {
+        this._stats.videoFps?.end();
+        this._stats.total?.end();
+        this._stats.memory?.end();
+      }
+
+      this._isDrawing = false;
+    } catch {
+      this._sendRenderingError(new DrawFrameError('Failed to draw frame'));
+    }
+  }
+
+  private _drawWatermark(form: CanvasForm, frameBitmap: ImageBitmap): void {
+    const frameWidth = this._canvas?.width || frameBitmap.width;
+    const frameHeight = this._canvas?.height || frameBitmap.height;
+    // Font size is either 12 or smaller based on available width
+    // since the font is not monospaced, we approximate it'll fit 1.5 more characters than monospaced
+    const approximateFontSize = Math.min(
+      Math.floor(frameWidth / (VIDEO_WATERMARK_TEXT.length / 1.5)),
+      12,
+    );
+
+    form.ctx.font = `${approximateFontSize}px "Inter", sans-serif`;
+    const measureGeneratedBy = form.ctx.measureText(VIDEO_WATERMARK_TEXT);
+
+    const textBoxWidth =
+      measureGeneratedBy.width + 2 * WATERMARK_BOX_HORIZONTAL_PADDING;
+    const textBoxHeight =
+      measureGeneratedBy.actualBoundingBoxAscent +
+      2 * WATERMARK_BOX_VERTICAL_PADDING;
+    const textBoxX = frameWidth - textBoxWidth;
+    const textBoxY = frameHeight - textBoxHeight;
+
+    form.ctx.fillStyle = 'rgba(0, 0, 0, 0.4)';
+    form.ctx.beginPath();
+    form.ctx.roundRect(
+      Math.round(textBoxX),
+      Math.round(textBoxY),
+      Math.round(textBoxWidth),
+      Math.round(textBoxHeight),
+      [WATERMARK_BOX_HORIZONTAL_PADDING, 0, 0, 0],
+    );
+    form.ctx.fill();
+
+    // Always reset the text style because some effects may change text styling in the same ctx
+    form.ctx.fillStyle = 'rgba(255, 255, 255, 0.8)';
+    form.ctx.textAlign = 'left';
+
+    form.ctx.fillText(
+      VIDEO_WATERMARK_TEXT,
+      Math.round(textBoxX + WATERMARK_BOX_HORIZONTAL_PADDING),
+      Math.round(
+        textBoxY +
+          WATERMARK_BOX_VERTICAL_PADDING +
+          measureGeneratedBy.actualBoundingBoxAscent,
+      ),
+    );
+  }
+
+  private updateFrameIndex(index: number): void {
+    this._frameIndex = index;
+    this.sendResponse<FrameUpdateResponse>('frameUpdate', {
+      index,
+    });
+  }
+
+  private _loadWatermarkFonts() {
+    const requiredFonts = [
+      {
+        url: '/fonts/Inter-VariableFont.ttf',
+        format: 'truetype-variations',
+      },
+    ];
+    requiredFonts.forEach(requiredFont => {
+      const fontFace = new FontFace(
+        'Inter',
+        `url(${requiredFont.url}) format('${requiredFont.format}')`,
+      );
+      fontFace.load().then(font => {
+        self.fonts.add(font);
+      });
+    });
+  }
+
+  private updatePlayback(playing: boolean): void {
+    if (playing) {
+      this.sendResponse<PlayRequest>('play');
+    } else {
+      this.sendResponse<PauseRequest>('pause');
+    }
+    this._isPlaying = playing;
+  }
+
+  private _cancelRender(): void {
+    if (this._playbackTimeoutHandle !== null) {
+      clearTimeout(this._playbackTimeoutHandle);
+      this._playbackTimeoutHandle = null;
+    }
+    if (this._playbackRAFHandle !== null) {
+      cancelAnimationFrame(this._playbackRAFHandle);
+      this._playbackRAFHandle = null;
+    }
+  }
+
+  private _sendRenderingError(error: Error): void {
+    this.sendResponse<RenderingErrorResponse>('renderingError', {
+      error: serializeError(error),
+    });
+  }
+
+  private _processEffects(
+    form: CanvasForm,
+    effectParams: EffectFrameContext,
+    tracklets: Tracklet[],
+  ) {
+    for (let i = 0; i < this._effects.length; i++) {
+      const effect = this._effects[i];
+
+      if (i === 0) {
+        this._stats.effect0?.begin();
+      } else if (i === 1) {
+        this._stats.effect1?.begin();
+      }
+
+      effect.apply(form, effectParams, tracklets);
+
+      if (i === 0) {
+        this._stats.effect0?.end();
+      } else if (i === 1) {
+        this._stats.effect1?.end();
+      }
+    }
+  }
+}

demo/frontend/src/common/components/video/VideoWorkerTypes.ts

@@ -0,0 +1,143 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodeEvent,
+  EffectUpdateEvent,
+  EncodingCompletedEvent,
+  EncodingStateUpdateEvent,
+  FilmstripEvent,
+  FrameUpdateEvent,
+  LoadStartEvent,
+  RenderingErrorEvent,
+} from './VideoWorkerBridge';
+import {EffectOptions} from './effects/Effect';
+import type {Effects} from './effects/Effects';
+
+export type Request<A, P> = {
+  action: A;
+} & P;
+
+// REQUESTS
+
+export type SetCanvasRequest = Request<
+  'setCanvas',
+  {
+    canvas: OffscreenCanvas;
+  }
+>;
+export type SetSourceRequest = Request<
+  'setSource',
+  {
+    source: string;
+  }
+>;
+export type PlayRequest = Request<'play', unknown>;
+export type PauseRequest = Request<'pause', unknown>;
+export type StopRequest = Request<'stop', unknown>;
+export type FrameUpdateRequest = Request<
+  'frameUpdate',
+  {
+    index: number;
+  }
+>;
+export type FilmstripRequest = Request<
+  'filmstrip',
+  {
+    width: number;
+    height: number;
+  }
+>;
+export type SetEffectRequest = Request<
+  'setEffect',
+  {
+    name: keyof Effects;
+    index: number;
+    options?: EffectOptions;
+  }
+>;
+
+export type EncodeVideoRequest = Request<'encode', unknown>;
+
+export type EnableStatsRequest = Request<'enableStats', unknown>;
+
+export type VideoWorkerRequest =
+  | SetCanvasRequest
+  | SetSourceRequest
+  | PlayRequest
+  | PauseRequest
+  | StopRequest
+  | FrameUpdateRequest
+  | FilmstripRequest
+  | SetEffectRequest
+  | EncodeVideoRequest
+  | EnableStatsRequest;
+
+export type VideoWorkerRequestMessageEvent = MessageEvent<VideoWorkerRequest>;
+
+// RESPONSES
+
+export type ErrorResponse = Request<
+  'error',
+  {
+    error: unknown;
+  }
+>;
+
+export type DecodeResponse = Request<'decode', DecodeEvent>;
+
+export type EncodingStateUpdateResponse = Request<
+  'encodingStateUpdate',
+  EncodingStateUpdateEvent
+>;
+
+export type EncodingCompletedResponse = Request<
+  'encodingCompleted',
+  EncodingCompletedEvent
+>;
+
+export type FilmstripResponse = Request<'filmstrip', FilmstripEvent>;
+
+export type PlayResponse = Request<'play', unknown>;
+
+export type PauseResponse = Request<'pause', unknown>;
+
+export type FrameUpdateResponse = Request<'frameUpdate', FrameUpdateEvent>;
+
+export type RenderingErrorResponse = Request<
+  'renderingError',
+  RenderingErrorEvent
+>;
+
+// HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+
+export type LoadStartResponse = Request<'loadstart', LoadStartEvent>;
+
+export type EffectUpdateResponse = Request<'effectUpdate', EffectUpdateEvent>;
+
+export type VideoWorkerResponse =
+  | ErrorResponse
+  | FilmstripResponse
+  | DecodeResponse
+  | EncodingStateUpdateResponse
+  | EncodingCompletedResponse
+  | PlayResponse
+  | PauseResponse
+  | FrameUpdateResponse
+  | LoadStartResponse
+  | RenderingErrorResponse
+  | EffectUpdateResponse;
+
+export type VideoWorkerResponseMessageEvent = MessageEvent<VideoWorkerResponse>;

demo/frontend/src/common/components/video/editor/DemoVideoEditor.tsx

@@ -0,0 +1,310 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletsAnnotation from '@/common/components/annotations/TrackletsAnnotation';
+import useCloseSessionBeforeUnload from '@/common/components/session/useCloseSessionBeforeUnload';
+import MessagesSnackbar from '@/common/components/snackbar/MessagesSnackbar';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import VideoFilmstripWithPlayback from '@/common/components/video/VideoFilmstripWithPlayback';
+import {
+  FrameUpdateEvent,
+  RenderingErrorEvent,
+  SessionStartedEvent,
+  TrackletsEvent,
+} from '@/common/components/video/VideoWorkerBridge';
+import VideoEditor from '@/common/components/video/editor/VideoEditor';
+import useResetDemoEditor from '@/common/components/video/editor/useResetEditor';
+import useVideo from '@/common/components/video/editor/useVideo';
+import InteractionLayer from '@/common/components/video/layers/InteractionLayer';
+import {PointsLayer} from '@/common/components/video/layers/PointsLayer';
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import UploadLoadingScreen from '@/common/loading/UploadLoadingScreen';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {
+  activeTrackletObjectIdAtom,
+  frameIndexAtom,
+  isAddObjectEnabledAtom,
+  isPlayingAtom,
+  isVideoLoadingAtom,
+  pointsAtom,
+  sessionAtom,
+  streamingStateAtom,
+  trackletObjectsAtom,
+  uploadingStateAtom,
+  VideoData,
+} from '@/demo/atoms';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {color, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useEffect, useState} from 'react';
+import type {ErrorObject} from 'serialize-error';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    overflow: 'auto',
+    width: '100%',
+    borderColor: color['gray-800'],
+    backgroundColor: color['gray-800'],
+    borderWidth: 8,
+    borderRadius: 12,
+    '@media screen and (max-width: 768px)': {
+      // on mobile, we want to grow the editor container so that the editor
+      // fills the remaining vertical space between the navbar and bottom
+      // of the page
+      flexGrow: 1,
+      borderWidth: 0,
+      borderRadius: 0,
+      paddingBottom: spacing[4],
+    },
+  },
+  loadingScreenWrapper: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    width: '100%',
+    height: '100%',
+    background: 'white',
+    overflow: 'hidden',
+    overflowY: 'auto',
+    zIndex: 999,
+  },
+});
+
+type Props = {
+  video: VideoData;
+};
+
+export default function DemoVideoEditor({video: inputVideo}: Props) {
+  const {settings} = useSettingsContext();
+  const video = useVideo();
+
+  const [isSessionStartFailed, setIsSessionStartFailed] =
+    useState<boolean>(false);
+
+  const [session, setSession] = useAtom(sessionAtom);
+
+  const [activeTrackletId, setActiveTrackletObjectId] = useAtom(
+    activeTrackletObjectIdAtom,
+  );
+  const setTrackletObjects = useSetAtom(trackletObjectsAtom);
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const points = useAtomValue(pointsAtom);
+  const isAddObjectEnabled = useAtomValue(isAddObjectEnabledAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isVideoLoading = useAtomValue(isVideoLoadingAtom);
+  const uploadingState = useAtomValue(uploadingStateAtom);
+
+  const [renderingError, setRenderingError] = useState<ErrorObject | null>(
+    null,
+  );
+
+  const {isMobile} = useScreenSize();
+
+  const [tabIndex] = useToolbarTabs();
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useCloseSessionBeforeUnload();
+
+  const {resetEditor, resetSession} = useResetDemoEditor();
+  useEffect(() => {
+    resetEditor();
+  }, [inputVideo, resetEditor]);
+
+  useEffect(() => {
+    function onFrameUpdate(event: FrameUpdateEvent) {
+      setFrameIndex(event.index);
+    }
+
+    // Listen to frame updates to fetch the frame index in the main thread,
+    // which is then used downstream to render points per frame.
+    video?.addEventListener('frameUpdate', onFrameUpdate);
+
+    function onSessionStarted(event: SessionStartedEvent) {
+      setSession({id: event.sessionId, ranPropagation: false});
+    }
+
+    video?.addEventListener('sessionStarted', onSessionStarted);
+
+    function onSessionStartFailed() {
+      setIsSessionStartFailed(true);
+    }
+
+    video?.addEventListener('sessionStartFailed', onSessionStartFailed);
+
+    function onTrackletsUpdated(event: TrackletsEvent) {
+      const tracklets = event.tracklets;
+      if (tracklets.length === 0) {
+        resetSession();
+      }
+      setTrackletObjects(tracklets);
+    }
+
+    video?.addEventListener('trackletsUpdated', onTrackletsUpdated);
+
+    function onRenderingError(event: RenderingErrorEvent) {
+      setRenderingError(event.error);
+    }
+
+    video?.addEventListener('renderingError', onRenderingError);
+
+    video?.initializeTracker('SAM 2', {
+      inferenceEndpoint: settings.inferenceAPIEndpoint,
+    });
+
+    video?.startSession(inputVideo.path);
+
+    return () => {
+      video?.closeSession();
+      video?.removeEventListener('frameUpdate', onFrameUpdate);
+      video?.removeEventListener('sessionStarted', onSessionStarted);
+      video?.removeEventListener('sessionStartFailed', onSessionStartFailed);
+      video?.removeEventListener('trackletsUpdated', onTrackletsUpdated);
+      video?.removeEventListener('renderingError', onRenderingError);
+    };
+  }, [
+    setFrameIndex,
+    setSession,
+    setTrackletObjects,
+    resetSession,
+    inputVideo,
+    video,
+    settings.inferenceAPIEndpoint,
+    settings.videoAPIEndpoint,
+  ]);
+
+  async function handleOptimisticPointUpdate(newPoints: SegmentationPoint[]) {
+    if (session == null) {
+      return;
+    }
+
+    async function createActiveTracklet() {
+      if (!isAddObjectEnabled || newPoints.length === 0) {
+        return;
+      }
+      const tracklet = await video?.createTracklet();
+      if (tracklet != null && newPoints.length > 0) {
+        setActiveTrackletObjectId(tracklet.id);
+        video?.updatePoints(tracklet.id, [newPoints[newPoints.length - 1]]);
+      }
+    }
+
+    if (activeTrackletId != null) {
+      video?.updatePoints(activeTrackletId, newPoints);
+    } else {
+      await createActiveTracklet();
+    }
+    enqueueMessage('pointClick');
+  }
+
+  async function handleAddPoint(point: SegmentationPoint) {
+    if (streamingState === 'partial' || streamingState === 'requesting') {
+      return;
+    }
+    if (isPlaying) {
+      return video?.pause();
+    }
+    handleOptimisticPointUpdate([...points, point]);
+  }
+
+  function handleRemovePoint(point: SegmentationPoint) {
+    if (
+      isPlaying ||
+      streamingState === 'partial' ||
+      streamingState === 'requesting'
+    ) {
+      return;
+    }
+    handleOptimisticPointUpdate(points.filter(p => p !== point));
+  }
+
+  // The interaction layer handles clicks onto the video canvas. It is used
+  // to get absolute point clicks within the video's coordinate system.
+  // The PointsLayer handles rendering of input points and allows removing
+  // individual points by clicking on them.
+  const layers = (
+    <>
+      {tabIndex === OBJECT_TOOLBAR_INDEX && (
+        <>
+          <InteractionLayer
+            key="interaction-layer"
+            onPoint={point => handleAddPoint(point)}
+          />
+          <PointsLayer
+            key="points-layer"
+            points={points}
+            onRemovePoint={handleRemovePoint}
+          />
+        </>
+      )}
+      {!isMobile && <MessagesSnackbar key="snackbar-layer" />}
+    </>
+  );
+
+  return (
+    <>
+      {(isVideoLoading || session === null) && !isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Loading demo..."
+            description="This may take a few moments, you're almost there!"
+          />
+        </div>
+      )}
+      {isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Did we just break the internet?"
+            description={
+              <>Uh oh, it looks like there was an issue starting a session.</>
+            }
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {isMobile && renderingError != null && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Well, this is embarrassing..."
+            description="This demo is not optimized for your device. Please try again on a different device with a larger screen."
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {uploadingState !== 'default' && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <UploadLoadingScreen />
+        </div>
+      )}
+      <div {...stylex.props(styles.container)}>
+        <VideoEditor
+          video={inputVideo}
+          layers={layers}
+          loading={session == null}>
+          <div className="bg-graydark-800 w-full">
+            <VideoFilmstripWithPlayback />
+            <TrackletsAnnotation />
+          </div>
+        </VideoEditor>
+      </div>
+    </>
+  );
+}

demo/frontend/src/common/components/video/editor/ImageUtils.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function convertVideoFrameToImageData(
+  videoFrame: VideoFrame,
+): ImageData | undefined {
+  const canvas = new OffscreenCanvas(
+    videoFrame.displayWidth,
+    videoFrame.displayHeight,
+  );
+  const ctx = canvas.getContext('2d');
+  ctx?.drawImage(videoFrame, 0, 0);
+  return ctx?.getImageData(0, 0, canvas.width, canvas.height);
+}
+
+/**
+ * This utility provides two functions:
+ * `process`: to find the bounding box of non-empty pixels from an ImageData, when looping through all its pixels
+ * `crop` to cut out the subsection found in `process`
+ * @returns
+ */
+export function findBoundingBox() {
+  let xMin = Number.MAX_VALUE;
+  let yMin = Number.MAX_VALUE;
+  let xMax = Number.MIN_VALUE;
+  let yMax = Number.MIN_VALUE;
+  return {
+    process: function (x: number, y: number, hasData: boolean) {
+      if (hasData) {
+        xMin = Math.min(x, xMin);
+        xMax = Math.max(x, xMax);
+        yMin = Math.min(y, yMin);
+        yMax = Math.max(y, yMax);
+      }
+      return [xMin, xMax, yMin, yMax];
+    },
+    crop(imageData: ImageData): ImageData | null {
+      const canvas = new OffscreenCanvas(imageData.width, imageData.height);
+      const ctx = canvas.getContext('2d');
+
+      const boundingBoxWidth = xMax - xMin;
+      const boundingBoxHeight = yMax - yMin;
+      if (ctx && boundingBoxWidth > 0 && boundingBoxHeight > 0) {
+        ctx.clearRect(0, 0, canvas.width, canvas.height);
+        ctx.putImageData(imageData, 0, 0);
+        return ctx.getImageData(
+          xMin,
+          yMin,
+          boundingBoxWidth,
+          boundingBoxHeight,
+        );
+      } else {
+        return null;
+      }
+    },
+    getBox(): [[number, number], [number, number]] {
+      return [
+        [xMin, yMin],
+        [xMax, yMax],
+      ];
+    },
+  };
+}
+
+export function magnifyImageRegion(
+  canvas: HTMLCanvasElement | null,
+  x: number,
+  y: number,
+  radius: number = 25,
+  scale: number = 2,
+): string {
+  if (canvas == null) {
+    return '';
+  }
+  const ctx = canvas.getContext('2d');
+  if (ctx) {
+    const minX = x - radius < 0 ? radius - x : 0;
+    const minY = y - radius < 0 ? radius - y : 0;
+    const region = ctx.getImageData(
+      Math.max(x - radius, 0),
+      Math.max(y - radius, 0),
+      radius * 2,
+      radius * 2,
+    );
+
+    // ImageData doesn't scale-transform correctly on canvas
+    // So we first draw the original size on an offscreen canvas, and then scale it
+    const regionCanvas = new OffscreenCanvas(region.width, region.height);
+    const regionCtx = regionCanvas.getContext('2d');
+    regionCtx?.putImageData(region, minX > 0 ? minX : 0, minY > 0 ? minY : 0);
+
+    const scaleCanvas = document.createElement('canvas');
+    scaleCanvas.width = Math.round(region.width * scale);
+    scaleCanvas.height = Math.round(region.height * scale);
+    const scaleCtx = scaleCanvas.getContext('2d');
+    scaleCtx?.scale(scale, scale);
+    scaleCtx?.drawImage(regionCanvas, 0, 0);
+
+    return scaleCanvas.toDataURL();
+  }
+  return '';
+}

demo/frontend/src/common/components/video/editor/VideoEditor.tsx

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {VideoData} from '@/demo/atoms';
+import stylex, {StyleXStyles} from '@stylexjs/stylex';
+import {useSetAtom} from 'jotai';
+import {PropsWithChildren, RefObject, useEffect, useRef} from 'react';
+import Video, {VideoRef} from '../Video';
+import {videoAtom} from './atoms';
+
+const MAX_VIDEO_WIDTH = 1280;
+
+const styles = stylex.create({
+  editorContainer: {
+    position: 'relative',
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    width: '100%',
+    height: '100%',
+    borderRadius: '0.375rem',
+    overflow: {
+      default: 'clip',
+      '@media screen and (max-width: 768px)': 'visible',
+    },
+  },
+  videoContainer: {
+    position: 'relative',
+    flexGrow: 1,
+    overflow: 'hidden',
+    width: '100%',
+    maxWidth: MAX_VIDEO_WIDTH,
+  },
+  layers: {
+    position: 'absolute',
+    left: 0,
+    top: 0,
+    bottom: 0,
+    right: 0,
+  },
+  loadingMessage: {
+    position: 'absolute',
+    top: '8px',
+    right: '8px',
+    padding: '6px 10px',
+    backgroundColor: '#6441D2CC',
+    color: '#FFF',
+    display: 'flex',
+    alignItems: 'center',
+    gap: '8px',
+    borderRadius: '8px',
+    fontSize: '0.8rem',
+  },
+});
+
+export type InteractionLayerProps = {
+  style: StyleXStyles;
+  videoRef: RefObject<VideoRef>;
+};
+
+export type ControlsProps = {
+  isPlaying: boolean;
+  onPlay: () => void;
+  onPause: () => void;
+  onPreviousFrame?: () => void;
+  onNextFrame?: () => void;
+};
+
+type Props = PropsWithChildren<{
+  video: VideoData;
+  layers?: React.ReactNode;
+  loading?: boolean;
+}>;
+
+export default function VideoEditor({
+  video: inputVideo,
+  layers,
+  loading,
+  children,
+}: Props) {
+  const videoRef = useRef<VideoRef>(null);
+  const setVideo = useSetAtom(videoAtom);
+
+  // Initialize video atom
+  useEffect(() => {
+    setVideo(videoRef.current);
+    return () => {
+      setVideo(null);
+    };
+  }, [setVideo]);
+
+  return (
+    <div {...stylex.props(styles.editorContainer)}>
+      <div {...stylex.props(styles.videoContainer)}>
+        <Video
+          ref={videoRef}
+          src={inputVideo.url}
+          width={inputVideo.width}
+          height={inputVideo.height}
+          loading={loading}
+        />
+        <div {...stylex.props(styles.layers)}>{layers}</div>
+      </div>
+      {children}
+    </div>
+  );
+}

demo/frontend/src/common/components/video/editor/VideoEditorUtils.ts

@@ -0,0 +1,253 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Mask, Tracklet} from '@/common/tracker/Tracker';
+import {
+  convertVideoFrameToImageData,
+  findBoundingBox,
+} from '@/common/utils/ImageUtils';
+import {DataArray} from '@/jscocotools/mask';
+import invariant from 'invariant';
+
+function getCanvas(
+  width: number,
+  height: number,
+  isOffscreen: boolean = false,
+): HTMLCanvasElement | OffscreenCanvas {
+  if (isOffscreen || typeof document === 'undefined') {
+    return new OffscreenCanvas(width, height);
+  }
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  return canvas;
+}
+
+export function drawFrame(
+  ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+  frame: VideoFrame | HTMLImageElement,
+  width: number,
+  height: number,
+) {
+  ctx?.drawImage(frame, 0, 0, width, height);
+}
+
+/**
+ * Given a mask and the image frame, get the masked image cropped to its bounding box.
+ */
+export function getThumbnailImageDataOld(
+  mask: DataArray,
+  videoFrame: VideoFrame,
+): ImageData | null {
+  const data = mask.data;
+  if (!ArrayBuffer.isView(data) || !(data instanceof Uint8Array)) {
+    return new ImageData(0, 0);
+  }
+
+  const frame = convertVideoFrameToImageData(videoFrame);
+  if (!frame) {
+    return new ImageData(0, 0);
+  }
+
+  const frameData = frame.data;
+  const scaleX = frame.width / mask.shape[1];
+  const scaleY = frame.height / mask.shape[0];
+  const boundingBox = findBoundingBox();
+  const transformedData = new Uint8ClampedArray(data.length * 4);
+
+  for (let i = 0; i < data.length; i++) {
+    // Since the mask is rotated, new width is the mask's height = mask.shape[1];
+    // Transform matrix: doing a rotate 90deg and then flip horizontal is the same as flipping x and y
+    // [ 0 1 ]   [ -1 0 ]   =   [ 0 1 ]
+    // [-1 0 ] x [  0 1 ]   =   [ 1 0 ]
+    // So, we can find the new index as: newY * newWidth + newX
+    const newX = Math.floor(i / mask.shape[0]); // ie, new x is the current y
+    const newY = i % mask.shape[0];
+    const transformedIndex = (newY * mask.shape[1] + newX) * 4;
+    const frameDataIndex = (newY * mask.shape[1] * scaleY + newX * scaleX) * 4;
+
+    transformedData[transformedIndex] = frameData[frameDataIndex];
+    transformedData[transformedIndex + 1] = frameData[frameDataIndex + 1];
+    transformedData[transformedIndex + 2] = frameData[frameDataIndex + 2];
+    transformedData[transformedIndex + 3] = (data[i] && 255) || 0; // A value
+
+    boundingBox.process(newX, newY, data[i] > 0);
+  }
+
+  const rotatedData = new ImageData(
+    transformedData,
+    mask.shape[1],
+    mask.shape[0],
+  ); // flip w and h of the mask
+
+  return boundingBox.crop(rotatedData);
+}
+
+/**
+ * Given a mask, the mask rendering context, and the video frame, get the
+ * masked image cropped to its bounding box.
+ */
+function getThumbnailImageData(
+  mask: Mask,
+  maskCtx: OffscreenCanvasRenderingContext2D,
+  frameBitmap: ImageBitmap,
+): ImageData | null {
+  const x = mask.bounds[0][0];
+  const y = mask.bounds[0][1];
+  const w = mask.bounds[1][0] - mask.bounds[0][0];
+  const h = mask.bounds[1][1] - mask.bounds[0][1];
+
+  if (w <= 0 || h <= 0) {
+    return null;
+  }
+
+  const thumbnailMaskData = maskCtx.getImageData(x, y, w, h);
+
+  const canvas = new OffscreenCanvas(w, h);
+  const ctx = canvas.getContext('2d');
+  invariant(ctx !== null, '2d context cannot be null');
+
+  ctx.putImageData(thumbnailMaskData, 0, 0);
+  ctx.globalCompositeOperation = 'source-in';
+  ctx.drawImage(frameBitmap, x, y, w, h, 0, 0, w, h);
+
+  return ctx.getImageData(0, 0, w, h);
+}
+
+export async function generateThumbnail(
+  track: Tracklet,
+  frameIndex: number,
+  mask: Mask,
+  frame: VideoFrame,
+  ctx: OffscreenCanvasRenderingContext2D,
+): Promise<void> {
+  // If a frame doesn't have points, the points will be undefined.
+  const hasPoints = (track.points[frameIndex]?.length ?? 0) > 0;
+  if (!hasPoints) {
+    return;
+  }
+  invariant(frame !== null, 'frame must be ready');
+  const bitmap = await createImageBitmap(frame);
+  const thumbnailImageData = getThumbnailImageData(
+    mask,
+    ctx as OffscreenCanvasRenderingContext2D,
+    bitmap,
+  );
+
+  bitmap.close();
+  if (thumbnailImageData != null) {
+    const thumbnailDataURL = await getDataURLFromImageData(thumbnailImageData);
+    track.thumbnail = thumbnailDataURL;
+  }
+}
+
+export async function getDataURLFromImageData(
+  imageData: ImageData | null,
+): Promise<string> {
+  if (!imageData) {
+    return '';
+  }
+
+  const canvas = getCanvas(imageData.width, imageData.height);
+  const ctx = canvas.getContext('2d');
+
+  if (ctx === null) {
+    return '';
+  }
+
+  ctx?.putImageData(imageData, 0, 0);
+
+  if (canvas instanceof OffscreenCanvas) {
+    const blob = await canvas.convertToBlob();
+    return new Promise(resolve => {
+      const reader = new FileReader();
+      reader.addEventListener(
+        'load',
+        () => {
+          const result = reader.result;
+          if (typeof result === 'string') {
+            resolve(result);
+          } else {
+            resolve('');
+          }
+        },
+        false,
+      );
+      reader.readAsDataURL(blob);
+    });
+  }
+  return canvas.toDataURL();
+}
+
+export function hexToRgb(hex: string): {
+  r: number;
+  g: number;
+  b: number;
+  a: number;
+} {
+  const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})?$/i.exec(
+    hex,
+  );
+  return result
+    ? {
+        r: parseInt(result[1], 16),
+        g: parseInt(result[2], 16),
+        b: parseInt(result[3], 16),
+        a: result[4] != null ? parseInt(result[4], 16) : 128,
+      }
+    : {r: 255, g: 0, b: 0, a: 128};
+}
+
+export function getPointInImage(
+  event: React.MouseEvent<HTMLElement>,
+  canvas: HTMLCanvasElement,
+  normalized: boolean = false,
+): [x: number, y: number] {
+  const rect = canvas.getBoundingClientRect();
+
+  const matrix = new DOMMatrix();
+
+  // First, center the image
+  const elementCenter = new DOMPoint(
+    canvas.clientWidth / 2,
+    canvas.clientHeight / 2,
+  );
+  const imageCenter = new DOMPoint(canvas.width / 2, canvas.height / 2);
+  matrix.translateSelf(
+    elementCenter.x - imageCenter.x,
+    elementCenter.y - imageCenter.y,
+  );
+
+  // Containing the object take the minimal scale
+  const scale = Math.min(
+    canvas.clientWidth / canvas.width,
+    canvas.clientHeight / canvas.height,
+  );
+  matrix.scaleSelf(scale, scale, 1, imageCenter.x, imageCenter.y);
+
+  const point = new DOMPoint(
+    event.clientX - rect.left,
+    event.clientY - rect.top,
+  );
+  const imagePoint = matrix.inverse().transformPoint(point);
+
+  const x = Math.max(Math.min(imagePoint.x, canvas.width), 0);
+  const y = Math.max(Math.min(imagePoint.y, canvas.height), 0);
+
+  if (normalized) {
+    return [x / canvas.width, y / canvas.height];
+  }
+  return [x, y];
+}

demo/frontend/src/common/components/video/editor/atoms.ts

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+import {VideoRef} from '../Video';
+
+export const videoAtom = atom<VideoRef | null>(null);

demo/frontend/src/common/components/video/editor/useResetEditor.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  activeTrackletObjectIdAtom,
+  frameIndexAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+  sessionAtom,
+  streamingStateAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {DEFAULT_EFFECT_LAYERS} from '@/demo/DemoConfig';
+import {useSetAtom} from 'jotai';
+import {useCallback} from 'react';
+
+type State = {
+  resetEditor: () => void;
+  resetEffects: () => void;
+  resetSession: () => void;
+};
+
+export default function useResetEditor(): State {
+  const video = useVideo();
+
+  const setSession = useSetAtom(sessionAtom);
+  const setActiveTrackletObjectId = useSetAtom(activeTrackletObjectIdAtom);
+  const setTrackletObjects = useSetAtom(trackletObjectsAtom);
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const setStreamingState = useSetAtom(streamingStateAtom);
+  const setIsPlaying = useSetAtom(isPlayingAtom);
+  const setIsStreaming = useSetAtom(isStreamingAtom);
+  const [, setDemoTabIndex] = useToolbarTabs();
+
+  const resetEffects = useCallback(() => {
+    video?.setEffect(DEFAULT_EFFECT_LAYERS.background, 0, {variant: 0});
+    video?.setEffect(DEFAULT_EFFECT_LAYERS.highlight, 1, {variant: 0});
+  }, [video]);
+
+  const resetEditor = useCallback(() => {
+    setFrameIndex(0);
+    setSession(null);
+    setActiveTrackletObjectId(0);
+    setTrackletObjects([]);
+    setStreamingState('none');
+    setIsPlaying(false);
+    setIsStreaming(false);
+    resetEffects();
+    setDemoTabIndex(OBJECT_TOOLBAR_INDEX);
+  }, [
+    setFrameIndex,
+    setSession,
+    setActiveTrackletObjectId,
+    setTrackletObjects,
+    setStreamingState,
+    setIsPlaying,
+    setIsStreaming,
+    resetEffects,
+    setDemoTabIndex,
+  ]);
+
+  const resetSession = useCallback(() => {
+    setSession(prev => {
+      if (prev === null) {
+        return prev;
+      }
+      return {...prev, ranPropagation: false};
+    });
+    setActiveTrackletObjectId(null);
+    resetEffects();
+  }, [setSession, setActiveTrackletObjectId, resetEffects]);
+
+  return {resetEditor, resetEffects, resetSession};
+}