init commit of samurai

sam2/demo/frontend/src/common/codecs/VideoDecoder.ts

@@ -0,0 +1,284 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {cloneFrame} from '@/common/codecs/WebCodecUtils';
+import {FileStream} from '@/common/utils/FileUtils';
+import {
+  createFile,
+  DataStream,
+  MP4ArrayBuffer,
+  MP4File,
+  MP4Sample,
+  MP4VideoTrack,
+} from 'mp4box';
+import {isAndroid, isChrome, isEdge, isWindows} from 'react-device-detect';
+
+export type ImageFrame = {
+  bitmap: VideoFrame;
+  timestamp: number;
+  duration: number;
+};
+
+export type DecodedVideo = {
+  width: number;
+  height: number;
+  frames: ImageFrame[];
+  numFrames: number;
+  fps: number;
+};
+
+function decodeInternal(
+  identifier: string,
+  onReady: (mp4File: MP4File) => Promise<void>,
+  onProgress: (decodedVideo: DecodedVideo) => void,
+): Promise<DecodedVideo> {
+  return new Promise((resolve, reject) => {
+    const imageFrames: ImageFrame[] = [];
+    const globalSamples: MP4Sample[] = [];
+
+    let decoder: VideoDecoder;
+
+    let track: MP4VideoTrack | null = null;
+    const mp4File = createFile();
+
+    mp4File.onError = reject;
+    mp4File.onReady = async info => {
+      if (info.videoTracks.length > 0) {
+        track = info.videoTracks[0];
+      } else {
+        // The video does not have a video track, so looking if there is an
+        // "otherTracks" available. Note, I couldn't find any documentation
+        // about "otherTracks" in WebCodecs [1], but it was available in the
+        // info for MP4V-ES, which isn't supported by Chrome [2].
+        // However, we'll still try to get the track and then throw an error
+        // further down in the VideoDecoder.isConfigSupported if the codec is
+        // not supported by the browser.
+        //
+        // [1] https://www.w3.org/TR/webcodecs/
+        // [2] https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs#mp4v-es
+        track = info.otherTracks[0];
+      }
+
+      if (track == null) {
+        reject(new Error(`${identifier} does not contain a video track`));
+        return;
+      }
+
+      const timescale = track.timescale;
+      const edits = track.edits;
+
+      let frame_n = 0;
+      decoder = new VideoDecoder({
+        // Be careful with any await in this function. The VideoDecoder will
+        // not await output and continue calling it with decoded frames.
+        async output(inputFrame) {
+          if (track == null) {
+            reject(new Error(`${identifier} does not contain a video track`));
+            return;
+          }
+
+          const saveTrack = track;
+
+          // If the track has edits, we'll need to check that only frames are
+          // returned that are within the edit list. This can happen for
+          // trimmed videos that have not been transcoded and therefore the
+          // video track contains more frames than those visually rendered when
+          // playing back the video.
+          if (edits != null && edits.length > 0) {
+            const cts = Math.round(
+              (inputFrame.timestamp * timescale) / 1_000_000,
+            );
+            if (cts < edits[0].media_time) {
+              inputFrame.close();
+              return;
+            }
+          }
+
+          // Workaround for Chrome where the decoding stops at ~17 frames unless
+          // the VideoFrame is closed. So, the workaround here is to create a
+          // new VideoFrame and close the decoded VideoFrame.
+          // The frame has to be cloned, or otherwise some frames at the end of the
+          // video will be black. Note, the default VideoFrame.clone doesn't work
+          // and it is using a frame cloning found here:
+          // https://webcodecs-blogpost-demo.glitch.me/
+          if (
+            (isAndroid && isChrome) ||
+            (isWindows && isChrome) ||
+            (isWindows && isEdge)
+          ) {
+            const clonedFrame = await cloneFrame(inputFrame);
+            inputFrame.close();
+            inputFrame = clonedFrame;
+          }
+
+          const sample = globalSamples[frame_n];
+          if (sample != null) {
+            const duration = (sample.duration * 1_000_000) / sample.timescale;
+            imageFrames.push({
+              bitmap: inputFrame,
+              timestamp: inputFrame.timestamp,
+              duration,
+            });
+            // Sort frames in order of timestamp. This is needed because Safari
+            // can return decoded frames out of order.
+            imageFrames.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
+            // Update progress on first frame and then every 40th frame
+            if (onProgress != null && frame_n % 100 === 0) {
+              onProgress({
+                width: saveTrack.track_width,
+                height: saveTrack.track_height,
+                frames: imageFrames,
+                numFrames: saveTrack.nb_samples,
+                fps:
+                  (saveTrack.nb_samples / saveTrack.duration) *
+                  saveTrack.timescale,
+              });
+            }
+          }
+          frame_n++;
+
+          if (saveTrack.nb_samples === frame_n) {
+            // Sort frames in order of timestamp. This is needed because Safari
+            // can return decoded frames out of order.
+            imageFrames.sort((a, b) => (a.timestamp > b.timestamp ? 1 : -1));
+            resolve({
+              width: saveTrack.track_width,
+              height: saveTrack.track_height,
+              frames: imageFrames,
+              numFrames: saveTrack.nb_samples,
+              fps:
+                (saveTrack.nb_samples / saveTrack.duration) *
+                saveTrack.timescale,
+            });
+          }
+        },
+        error(error) {
+          reject(error);
+        },
+      });
+
+      let description;
+      const trak = mp4File.getTrackById(track.id);
+      const entries = trak?.mdia?.minf?.stbl?.stsd?.entries;
+      if (entries == null) {
+        return;
+      }
+      for (const entry of entries) {
+        if (entry.avcC || entry.hvcC) {
+          const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
+          if (entry.avcC) {
+            entry.avcC.write(stream);
+          } else if (entry.hvcC) {
+            entry.hvcC.write(stream);
+          }
+          description = new Uint8Array(stream.buffer, 8); // Remove the box header.
+          break;
+        }
+      }
+
+      const configuration: VideoDecoderConfig = {
+        codec: track.codec,
+        codedWidth: track.track_width,
+        codedHeight: track.track_height,
+        description,
+      };
+      const supportedConfig =
+        await VideoDecoder.isConfigSupported(configuration);
+      if (supportedConfig.supported == true) {
+        decoder.configure(configuration);
+
+        mp4File.setExtractionOptions(track.id, null, {
+          nbSamples: Infinity,
+        });
+        mp4File.start();
+      } else {
+        reject(
+          new Error(
+            `Decoder config faile: config ${JSON.stringify(
+              supportedConfig.config,
+            )} is not supported`,
+          ),
+        );
+        return;
+      }
+    };
+
+    mp4File.onSamples = async (
+      _id: number,
+      _user: unknown,
+      samples: MP4Sample[],
+    ) => {
+      for (const sample of samples) {
+        globalSamples.push(sample);
+        decoder.decode(
+          new EncodedVideoChunk({
+            type: sample.is_sync ? 'key' : 'delta',
+            timestamp: (sample.cts * 1_000_000) / sample.timescale,
+            duration: (sample.duration * 1_000_000) / sample.timescale,
+            data: sample.data,
+          }),
+        );
+      }
+      await decoder.flush();
+      decoder.close();
+    };
+
+    onReady(mp4File);
+  });
+}
+
+export function decode(
+  file: File,
+  onProgress: (decodedVideo: DecodedVideo) => void,
+): Promise<DecodedVideo> {
+  return decodeInternal(
+    file.name,
+    async (mp4File: MP4File) => {
+      const reader = new FileReader();
+      reader.onload = function () {
+        const result = this.result as MP4ArrayBuffer;
+        if (result != null) {
+          result.fileStart = 0;
+          mp4File.appendBuffer(result);
+        }
+        mp4File.flush();
+      };
+      reader.readAsArrayBuffer(file);
+    },
+    onProgress,
+  );
+}
+
+export function decodeStream(
+  fileStream: FileStream,
+  onProgress: (decodedVideo: DecodedVideo) => void,
+): Promise<DecodedVideo> {
+  return decodeInternal(
+    'stream',
+    async (mp4File: MP4File) => {
+      let part = await fileStream.next();
+      while (part.done === false) {
+        const result = part.value.data.buffer as MP4ArrayBuffer;
+        if (result != null) {
+          result.fileStart = part.value.range.start;
+          mp4File.appendBuffer(result);
+        }
+        mp4File.flush();
+        part = await fileStream.next();
+      }
+    },
+    onProgress,
+  );
+}

sam2/demo/frontend/src/common/codecs/VideoEncoder.ts

@@ -0,0 +1,139 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {ImageFrame} from '@/common/codecs/VideoDecoder';
+import {MP4ArrayBuffer, createFile} from 'mp4box';
+
+// The selection of timescale and seconds/key-frame value are
+// explained in the following docs: https://github.com/vjeux/mp4-h264-re-encode
+const TIMESCALE = 90000;
+const SECONDS_PER_KEY_FRAME = 2;
+
+export function encode(
+  width: number,
+  height: number,
+  numFrames: number,
+  framesGenerator: AsyncGenerator<ImageFrame, unknown>,
+  progressCallback?: (progress: number) => void,
+): Promise<MP4ArrayBuffer> {
+  return new Promise((resolve, reject) => {
+    let encodedFrameIndex = 0;
+    let nextKeyFrameTimestamp = 0;
+    let trackID: number | null = null;
+    const durations: number[] = [];
+
+    const outputFile = createFile();
+
+    const encoder = new VideoEncoder({
+      output(chunk, metaData) {
+        const uint8 = new Uint8Array(chunk.byteLength);
+        chunk.copyTo(uint8);
+
+        const description = metaData?.decoderConfig?.description;
+        if (trackID === null) {
+          trackID = outputFile.addTrack({
+            width: width,
+            height: height,
+            timescale: TIMESCALE,
+            avcDecoderConfigRecord: description,
+          });
+        }
+        const shiftedDuration = durations.shift();
+        if (shiftedDuration != null) {
+          outputFile.addSample(trackID, uint8, {
+            duration: getScaledDuration(shiftedDuration),
+            is_sync: chunk.type === 'key',
+          });
+          encodedFrameIndex++;
+          progressCallback?.(encodedFrameIndex / numFrames);
+        }
+
+        if (encodedFrameIndex === numFrames) {
+          resolve(outputFile.getBuffer());
+        }
+      },
+      error(error) {
+        reject(error);
+        return;
+      },
+    });
+
+    const setConfigurationAndEncodeFrames = async () => {
+      // The codec value was taken from the following implementation and seems
+      // reasonable for our use case for now:
+      // https://github.com/vjeux/mp4-h264-re-encode/blob/main/mp4box.html#L103
+
+      // Additional details about codecs can be found here:
+      //  - https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
+      //  - https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry
+      //
+      // The following setting is a good compromise between output video file
+      // size and quality. The latencyMode "realtime" is needed for Safari,
+      // which otherwise will produce 20x larger files when in quality
+      // latencyMode. Chrome does a really good job with file size even when
+      // latencyMode is set to quality.
+      const configuration: VideoEncoderConfig = {
+        codec: 'avc1.4d0034',
+        width: roundToNearestEven(width),
+        height: roundToNearestEven(height),
+        bitrate: 14_000_000,
+        alpha: 'discard',
+        bitrateMode: 'variable',
+        latencyMode: 'realtime',
+      };
+      const supportedConfig =
+        await VideoEncoder.isConfigSupported(configuration);
+      if (supportedConfig.supported === true) {
+        encoder.configure(configuration);
+      } else {
+        throw new Error(
+          `Unsupported video encoder config ${JSON.stringify(supportedConfig)}`,
+        );
+      }
+
+      for await (const frame of framesGenerator) {
+        const {bitmap, duration, timestamp} = frame;
+        durations.push(duration);
+        let keyFrame = false;
+        if (timestamp >= nextKeyFrameTimestamp) {
+          await encoder.flush();
+          keyFrame = true;
+          nextKeyFrameTimestamp = timestamp + SECONDS_PER_KEY_FRAME * 1e6;
+        }
+        encoder.encode(bitmap, {keyFrame});
+        bitmap.close();
+      }
+
+      await encoder.flush();
+      encoder.close();
+    };
+
+    setConfigurationAndEncodeFrames();
+  });
+}
+
+function getScaledDuration(rawDuration: number) {
+  return rawDuration / (1_000_000 / TIMESCALE);
+}
+
+function roundToNearestEven(dim: number) {
+  const rounded = Math.round(dim);
+
+  if (rounded % 2 === 0) {
+    return rounded;
+  } else {
+    return rounded + (rounded > dim ? -1 : 1);
+  }
+}

sam2/demo/frontend/src/common/codecs/WebCodecUtils.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// https://github.com/w3c/webcodecs/issues/88
+// https://issues.chromium.org/issues/40725065
+// https://webcodecs-blogpost-demo.glitch.me/
+export async function cloneFrame(frame: VideoFrame): Promise<VideoFrame> {
+  const {
+    codedHeight,
+    codedWidth,
+    colorSpace,
+    displayHeight,
+    displayWidth,
+    format,
+    timestamp,
+  } = frame;
+  const rect = {x: 0, y: 0, width: codedWidth, height: codedHeight};
+  const data = new ArrayBuffer(frame.allocationSize({rect}));
+  try {
+    await frame.copyTo(data, {rect});
+  } catch (error) {
+    // The VideoFrame#copyTo on x64 builds on macOS fails. The workaround here
+    // is to clone the frame.
+    // https://stackoverflow.com/questions/77898766/inconsistent-behavior-of-webcodecs-copyto-method-across-different-browsers-an
+    return frame.clone();
+  }
+  return new VideoFrame(data, {
+    codedHeight,
+    codedWidth,
+    colorSpace,
+    displayHeight,
+    displayWidth,
+    duration: frame.duration ?? undefined,
+    format: format!,
+    timestamp,
+    visibleRect: frame.visibleRect ?? undefined,
+  });
+}

sam2/demo/frontend/src/common/components/MobileFirstClickBanner.tsx

@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideoModal from '@/common/components/gallery/ChangeVideoModal';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {spacing} from '@/theme/tokens.stylex';
+import {ImageCopy} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {Button} from 'react-daisyui';
+
+const styles = stylex.create({
+  container: {
+    position: 'relative',
+    backgroundColor: '#000',
+    padding: spacing[5],
+    paddingVertical: spacing[6],
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing[4],
+  },
+});
+
+export default function MobileFirstClickBanner() {
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div className="flex text-white text-lg">
+        Click an object in the video to start
+      </div>
+      <div className="text-sm text-[#A7B3BF]">
+        <p>
+          You&apos;ll be able to use {DEMO_SHORT_NAME} to make fun edits to any
+          video by tracking objects and applying visual effects. To start, click
+          any object in the video.
+        </p>
+      </div>
+      <div className="flex items-center">
+        <ChangeVideoModal
+          videoGalleryModalTrigger={MobileVideoGalleryModalTrigger}
+          showUploadInGallery={true}
+        />
+      </div>
+    </div>
+  );
+}
+
+type MobileVideoGalleryModalTriggerProps = {
+  onClick: () => void;
+};
+
+function MobileVideoGalleryModalTrigger({
+  onClick,
+}: MobileVideoGalleryModalTriggerProps) {
+  return (
+    <Button
+      color="ghost"
+      startIcon={<ImageCopy size={20} />}
+      onClick={onClick}
+      className="text-white p-0">
+      Change video
+    </Button>
+  );
+}

sam2/demo/frontend/src/common/components/Tooltip.tsx

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {PropsWithChildren} from 'react';
+
+type Props = PropsWithChildren<{
+  className?: string;
+  message: string;
+  position?: 'left' | 'top' | 'right' | 'bottom';
+}>;
+
+/**
+ * This is a custom Tooltip component because React Daisy UI does not have an
+ * option to *only* show tooltip on large devices.
+ */
+export default function Tooltip({
+  children,
+  className = '',
+  message,
+  position = 'top',
+}: Props) {
+  return (
+    <div
+      className={`lg:tooltip tooltip-${position} ${className}`}
+      data-tip={message}>
+      {children}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/AddObjectButton.tsx

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {activeTrackletObjectIdAtom, labelTypeAtom} from '@/demo/atoms';
+import {Add} from '@carbon/icons-react';
+import {useSetAtom} from 'jotai';
+
+export default function AddObjectButton() {
+  const video = useVideo();
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+  const setLabelType = useSetAtom(labelTypeAtom);
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  async function addObject() {
+    enqueueMessage('addObjectClick');
+    const tracklet = await video?.createTracklet();
+    if (tracklet != null) {
+      setActiveTrackletId(tracklet.id);
+      setLabelType('positive');
+    }
+  }
+
+  return (
+    <div
+      onClick={addObject}
+      className="group flex justify-start mx-4 px-4 bg-transparent text-white !rounded-xl border-none cursor-pointer">
+      <div className="flex gap-6 items-center">
+        <div className=" group-hover:bg-graydark-700 border border-white relative h-12 w-12 md:w-20 md:h-20 shrink-0 rounded-lg flex items-center justify-center">
+          <Add size={36} className="group-hover:text-white text-gray-300" />
+        </div>
+        <div className="font-medium text-base">Add another object</div>
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ClearAllPointsInVideoButton.tsx

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useRestartSession from '@/common/components/session/useRestartSession';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {isPlayingAtom, isStreamingAtom, labelTypeAtom} from '@/demo/atoms';
+import {Reset} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {useState} from 'react';
+import {Button, Loading} from 'react-daisyui';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+  },
+});
+
+type Props = {
+  onRestart: () => void;
+};
+
+export default function ClearAllPointsInVideoButton({onRestart}: Props) {
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setLabelType = useSetAtom(labelTypeAtom);
+  const {clearMessage} = useMessagesSnackbar();
+  const {restartSession} = useRestartSession();
+
+  const video = useVideo();
+
+  async function handleRestart() {
+    if (video === null) {
+      return;
+    }
+
+    setIsLoading(true);
+    if (isPlaying) {
+      video.pause();
+    }
+    if (isStreaming) {
+      await video.abortStreamMasks();
+    }
+    const isSuccessful = await video.clearPointsInVideo();
+    if (!isSuccessful) {
+      await restartSession();
+    }
+    video.frame = 0;
+    setLabelType('positive');
+    onRestart();
+    clearMessage();
+    setIsLoading(false);
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <Button
+        color="ghost"
+        onClick={handleRestart}
+        className="!px-4 !rounded-full font-medium text-white hover:bg-black"
+        startIcon={isLoading ? <Loading size="sm" /> : <Reset size={20} />}>
+        Start over
+      </Button>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/CloseSessionButton.tsx

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {ChevronRight} from '@carbon/icons-react';
+
+type Props = {
+  onSessionClose: () => void;
+};
+
+export default function CloseSessionButton({onSessionClose}: Props) {
+  const video = useVideo();
+
+  function handleCloseSession() {
+    video?.closeSession();
+    video?.logAnnotations();
+    onSessionClose();
+  }
+
+  return (
+    <PrimaryCTAButton onClick={handleCloseSession} endIcon={<ChevronRight />}>
+      Good to go
+    </PrimaryCTAButton>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/FirstClickView.tsx

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideo from '@/common/components/gallery/ChangeVideoModal';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {useEffect, useRef} from 'react';
+
+export default function FirstClickView() {
+  const isFirstClickMessageShown = useRef(false);
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useEffect(() => {
+    if (!isFirstClickMessageShown.current) {
+      isFirstClickMessageShown.current = true;
+      enqueueMessage('firstClick');
+    }
+  }, [enqueueMessage]);
+
+  return (
+    <div className="w-full h-full flex flex-col p-8">
+      <div className="grow flex flex-col gap-6">
+        <h2 className="text-2xl">Click an object in the video to start</h2>
+        <p className="!text-gray-60">
+          You&apos;ll be able to use {DEMO_SHORT_NAME} to make fun edits to any
+          video by tracking objects and applying visual effects.
+        </p>
+        <p className="!text-gray-60">
+          To start, click any object in the video.
+        </p>
+      </div>
+      <div className="flex items-center">
+        <ChangeVideo />
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/LimitNotice.tsx

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {InformationFilled} from '@carbon/icons-react';
+
+export default function LimitNotice() {
+  return (
+    <div className="mt-6 gap-3 mx-6 flex items-center text-gray-400">
+      <div>
+        <InformationFilled size={32} />
+      </div>
+      <div className="text-sm leading-snug">
+        In this demo, you can track up to 3 objects, even though the SAM 2 model
+        does not have a limit.
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/MobileObjectsList.tsx

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ClearAllPointsInVideoButton from '@/common/components/annotations/ClearAllPointsInVideoButton';
+import ObjectThumbnail from '@/common/components/annotations/ObjectThumbnail';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import {activeTrackletObjectIdAtom, trackletObjectsAtom} from '@/demo/atoms';
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue, useSetAtom} from 'jotai';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    padding: spacing[5],
+    borderTop: '1px solid #DEE3E9',
+  },
+  trackletsContainer: {
+    flexGrow: 1,
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[5],
+  },
+});
+
+type Props = {
+  showActiveObject: () => void;
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileObjectsList({
+  showActiveObject,
+  onTabChange,
+}: Props) {
+  const tracklets = useAtomValue(trackletObjectsAtom);
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+
+  function handleSelectObject(tracklet: BaseTracklet) {
+    setActiveTrackletId(tracklet.id);
+    showActiveObject();
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.trackletsContainer)}>
+        {tracklets.map(tracklet => {
+          const {id, color, thumbnail} = tracklet;
+          return (
+            <ObjectThumbnail
+              key={id}
+              color={color}
+              thumbnail={thumbnail}
+              onClick={() => {
+                handleSelectObject(tracklet);
+              }}
+            />
+          );
+        })}
+      </div>
+      <ClearAllPointsInVideoButton
+        onRestart={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/MobileObjectsToolbar.tsx

@@ -0,0 +1,51 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import MobileObjectsToolbarHeader from '@/common/components/annotations/MobileObjectsToolbarHeader';
+import ObjectsToolbarBottomActions from '@/common/components/annotations/ObjectsToolbarBottomActions';
+import {getObjectLabel} from '@/common/components/annotations/ObjectUtils';
+import ToolbarObject from '@/common/components/annotations/ToolbarObject';
+import MobileFirstClickBanner from '@/common/components/MobileFirstClickBanner';
+import {activeTrackletObjectAtom, isFirstClickMadeAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileObjectsToolbar({onTabChange}: Props) {
+  const activeTracklet = useAtomValue(activeTrackletObjectAtom);
+  const isFirstClickMade = useAtomValue(isFirstClickMadeAtom);
+
+  if (!isFirstClickMade) {
+    return <MobileFirstClickBanner />;
+  }
+
+  return (
+    <div className="w-full">
+      <MobileObjectsToolbarHeader />
+      {activeTracklet != null && (
+        <ToolbarObject
+          label={getObjectLabel(activeTracklet)}
+          tracklet={activeTracklet}
+          isActive={true}
+          isMobile={true}
+        />
+      )}
+
+      <ObjectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/MobileObjectsToolbarHeader.tsx

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarProgressChip from '@/common/components/toolbar/ToolbarProgressChip';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function MobileObjectsToolbarHeader() {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  return (
+    <div className="w-full flex gap-4 items-center px-5 py-5">
+      <div className="grow text-sm text-white">
+        <ToolbarProgressChip />
+        {streamingState === 'full'
+          ? 'Review your selected objects across the video, and continue to edit if needed. Once everything looks good, press “Next” to continue.'
+          : isStreaming
+            ? 'Watch the video closely for any places where your objects aren’t tracked correctly. You can also stop tracking to make additional edits.'
+            : 'Edit your object selection with a few more clicks if needed. Press “Track objects” to track your objects throughout the video.'}
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ObjectActions.tsx

@@ -0,0 +1,116 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PointsToggle from '@/common/components/annotations/PointsToggle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useReportError from '@/common/error/useReportError';
+import {
+  activeTrackletObjectIdAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+} from '@/demo/atoms';
+import {
+  AddFilled,
+  Select_02,
+  SubtractFilled,
+  TrashCan,
+} from '@carbon/icons-react';
+import {useAtom, useAtomValue} from 'jotai';
+import {useState} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = {
+  objectId: number;
+  active: boolean;
+};
+
+function CustomButton({className, ...props}: ButtonProps) {
+  return (
+    <Button
+      size="sm"
+      color="ghost"
+      className={`font-medium border-none hover:bg-black  px-2 h-10 ${className}`}
+      {...props}>
+      {props.children}
+    </Button>
+  );
+}
+
+export default function ObjectActions({objectId, active}: Props) {
+  const [isRemovingObject, setIsRemovingObject] = useState<boolean>(false);
+  const [activeTrackId, setActiveTrackletId] = useAtom(
+    activeTrackletObjectIdAtom,
+  );
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const isPlaying = useAtom(isPlayingAtom);
+
+  const video = useVideo();
+  const reportError = useReportError();
+
+  async function handleRemoveObject(
+    event: React.MouseEvent<HTMLButtonElement>,
+  ) {
+    try {
+      event.stopPropagation();
+      setIsRemovingObject(true);
+      if (isStreaming) {
+        await video?.abortStreamMasks();
+      }
+      if (isPlaying) {
+        video?.pause();
+      }
+      await video?.deleteTracklet(objectId);
+    } catch (error) {
+      reportError(error);
+    } finally {
+      setIsRemovingObject(false);
+      if (activeTrackId === objectId) {
+        setActiveTrackletId(null);
+      }
+    }
+  }
+
+  return (
+    <div>
+      {active && (
+        <div className="text-sm mt-1 leading-snug text-gray-400 hidden md:block ml-2 md:mb-4">
+          Select <AddFilled size={14} className="inline" /> to add areas to the
+          object and <SubtractFilled size={14} className="inline" /> to remove
+          areas from the object in the video. Click on an existing point to
+          delete it.
+        </div>
+      )}
+
+      <div className="flex justify-between items-center md:mt-2 mt-0">
+        {active ? (
+          <PointsToggle />
+        ) : (
+          <>
+            <CustomButton startIcon={<Select_02 size={24} />}>
+              Edit selection
+            </CustomButton>
+            <CustomButton
+              loading={isRemovingObject}
+              onClick={handleRemoveObject}
+              startIcon={!isRemovingObject && <TrashCan size={24} />}>
+              <span className="hidden md:inline">Clear</span>
+            </CustomButton>
+          </>
+        )}
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ObjectPlaceholder.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+
+type Props = {
+  showPlus?: boolean;
+  onClick?: () => void;
+};
+
+export default function ObjectPlaceholder({showPlus = true, onClick}: Props) {
+  return (
+    <div
+      className={`relative ${BLUE_PINK_FILL_BR} h-12 w-12 md:h-20 md:w-20 shrink-0 rounded-lg`}
+      onClick={onClick}>
+      {showPlus && (
+        <div className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2">
+          <svg
+            xmlns="http://www.w3.org/2000/svg"
+            width="16"
+            height="16"
+            viewBox="0 0 16 16"
+            fill="none">
+            <path
+              d="M16 7H9V0H7V7H0V9H7V16H9V9H16V7Z"
+              fill="#667788"
+              fillOpacity={1}
+            />
+          </svg>
+        </div>
+      )}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ObjectThumbnail.tsx

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type Props = {
+  thumbnail: string | null;
+  color: string;
+  onClick?: () => void;
+};
+
+export default function ObjectThumbnail({thumbnail, color, onClick}: Props) {
+  return (
+    <div
+      className="relative h-12 w-12 md:w-20 md:h-20 shrink-0 p-2 rounded-lg bg-contain bg-no-repeat bg-center"
+      style={{
+        backgroundColor: color,
+      }}
+      onClick={onClick}>
+      <div
+        className="w-full h-full bg-contain bg-no-repeat bg-center"
+        style={{
+          backgroundImage: thumbnail == null ? 'none' : `url(${thumbnail})`,
+        }}></div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ObjectUtils.ts

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BaseTracklet} from '@/common/tracker/Tracker';
+
+export function getObjectLabel(tracklet: BaseTracklet) {
+  return `Object ${tracklet.id + 1}`;
+}

sam2/demo/frontend/src/common/components/annotations/ObjectsToolbar.tsx

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import AddObjectButton from '@/common/components/annotations/AddObjectButton';
+import FirstClickView from '@/common/components/annotations/FirstClickView';
+import LimitNotice from '@/common/components/annotations/LimitNotice';
+import ObjectsToolbarBottomActions from '@/common/components/annotations/ObjectsToolbarBottomActions';
+import ObjectsToolbarHeader from '@/common/components/annotations/ObjectsToolbarHeader';
+import {getObjectLabel} from '@/common/components/annotations/ObjectUtils';
+import ToolbarObject from '@/common/components/annotations/ToolbarObject';
+import {
+  activeTrackletObjectAtom,
+  activeTrackletObjectIdAtom,
+  isAddObjectEnabledAtom,
+  isFirstClickMadeAtom,
+  isTrackletObjectLimitReachedAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function ObjectsToolbar({onTabChange}: Props) {
+  const tracklets = useAtomValue(trackletObjectsAtom);
+  const activeTracklet = useAtomValue(activeTrackletObjectAtom);
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+  const isFirstClickMade = useAtomValue(isFirstClickMadeAtom);
+  const isObjectLimitReached = useAtomValue(isTrackletObjectLimitReachedAtom);
+  const isAddObjectEnabled = useAtomValue(isAddObjectEnabledAtom);
+
+  if (!isFirstClickMade) {
+    return <FirstClickView />;
+  }
+
+  return (
+    <div className="flex flex-col h-full">
+      <ObjectsToolbarHeader />
+      <div className="grow w-full overflow-y-auto">
+        {tracklets.map(tracklet => {
+          return (
+            <ToolbarObject
+              key={tracklet.id}
+              label={getObjectLabel(tracklet)}
+              tracklet={tracklet}
+              isActive={activeTracklet?.id === tracklet.id}
+              onClick={() => {
+                setActiveTrackletId(tracklet.id);
+              }}
+            />
+          );
+        })}
+        {isAddObjectEnabled && <AddObjectButton />}
+        {isObjectLimitReached && <LimitNotice />}
+      </div>
+      <ObjectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ObjectsToolbarBottomActions.tsx

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ClearAllPointsInVideoButton from '@/common/components/annotations/ClearAllPointsInVideoButton';
+import CloseSessionButton from '@/common/components/annotations/CloseSessionButton';
+import TrackAndPlayButton from '@/common/components/button/TrackAndPlayButton';
+import ToolbarBottomActionsWrapper from '@/common/components/toolbar/ToolbarBottomActionsWrapper';
+import {
+  EFFECT_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function ObjectsToolbarBottomActions({onTabChange}: Props) {
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  const isTrackingEnabled =
+    streamingState !== 'none' && streamingState !== 'full';
+
+  function handleSwitchToEffectsTab() {
+    onTabChange(EFFECT_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <ClearAllPointsInVideoButton
+        onRestart={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+      {isTrackingEnabled && <TrackAndPlayButton />}
+      {streamingState === 'full' && (
+        <CloseSessionButton onSessionClose={handleSwitchToEffectsTab} />
+      )}
+    </ToolbarBottomActionsWrapper>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ObjectsToolbarHeader.tsx

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function ObjectsToolbarHeader() {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  return (
+    <ToolbarHeaderWrapper
+      title={
+        streamingState === 'full'
+          ? 'Review tracked objects'
+          : isStreaming
+            ? 'Tracking objects'
+            : 'Select objects'
+      }
+      description={
+        streamingState === 'full'
+          ? 'Review your selected objects across the video, and continue to edit if needed. Once everything looks good, press “Next” to continue.'
+          : isStreaming
+            ? 'Watch the video closely for any places where your objects aren’t tracked correctly. You can also stop tracking to make additional edits.'
+            : 'Adjust the selection of your object, or add additional objects. Press “Track objects” to track your objects throughout the video.'
+      }
+      className="mb-8"
+    />
+  );
+}

sam2/demo/frontend/src/common/components/annotations/PointsToggle.tsx

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {labelTypeAtom} from '@/demo/atoms';
+import {AddFilled, SubtractFilled} from '@carbon/icons-react';
+import {useAtom} from 'jotai';
+
+export default function PointsToggle() {
+  const [labelType, setLabelType] = useAtom(labelTypeAtom);
+  const isPositive = labelType === 'positive';
+
+  const buttonStyle = (selected: boolean) =>
+    `btn-md bg-graydark-800 !text-white md:px-2 lg:px-4 py-0.5 ${selected ? `border border-white hover:bg-graydark-800` : `border-graydark-700 hover:bg-graydark-700`}`;
+
+  return (
+    <div className="flex items-center w-full md:ml-2">
+      <div className="join group grow gap-[1px]">
+        <button
+          className={`w-1/2  btn join-item text-white ${buttonStyle(isPositive)}`}
+          onClick={() => setLabelType('positive')}>
+          <AddFilled size={24} className="text-blue-500" /> Add
+        </button>
+        <button
+          className={`w-1/2 btn join-item text-red-700 ${buttonStyle(!isPositive)}`}
+          onClick={() => setLabelType('negative')}>
+          <SubtractFilled size={24} className="text-red-400" />
+          Remove
+        </button>
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/PrimaryCTAButton.tsx

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+  disabled?: boolean;
+  endIcon?: ReactNode;
+} & React.DOMAttributes<HTMLButtonElement>;
+
+export default function PrimaryCTAButton({
+  children,
+  disabled,
+  endIcon,
+  ...props
+}: Props) {
+  return (
+    <GradientBorder disabled={disabled}>
+      <button
+        className={`btn ${disabled && 'btn-disabled'} !rounded-full !bg-black !text-white !border-none`}
+        {...props}>
+        {children}
+        {endIcon != null && endIcon}
+      </button>
+    </GradientBorder>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ToolbarObject.tsx

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ObjectActions from '@/common/components/annotations/ObjectActions';
+import ObjectPlaceholder from '@/common/components/annotations/ObjectPlaceholder';
+import ObjectThumbnail from '@/common/components/annotations/ObjectThumbnail';
+import ToolbarObjectContainer from '@/common/components/annotations/ToolbarObjectContainer';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import emptyFunction from '@/common/utils/emptyFunction';
+import {activeTrackletObjectIdAtom} from '@/demo/atoms';
+import {useSetAtom} from 'jotai';
+
+type Props = {
+  label: string;
+  tracklet: BaseTracklet;
+  isActive: boolean;
+  isMobile?: boolean;
+  onClick?: () => void;
+  onThumbnailClick?: () => void;
+};
+
+export default function ToolbarObject({
+  label,
+  tracklet,
+  isActive,
+  isMobile = false,
+  onClick,
+  onThumbnailClick = emptyFunction,
+}: Props) {
+  const video = useVideo();
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+
+  async function handleCancelNewObject() {
+    try {
+      await video?.deleteTracklet(tracklet.id);
+    } catch (error) {
+      reportError(error);
+    } finally {
+      setActiveTrackletId(null);
+    }
+  }
+
+  if (!tracklet.isInitialized) {
+    return (
+      <ToolbarObjectContainer
+        alignItems="center"
+        isActive={isActive}
+        title="New object"
+        subtitle="No object is currently selected. Click an object in the video."
+        thumbnail={<ObjectPlaceholder showPlus={false} />}
+        isMobile={isMobile}
+        onClick={onClick}
+        onCancel={handleCancelNewObject}
+      />
+    );
+  }
+
+  return (
+    <ToolbarObjectContainer
+      isActive={isActive}
+      onClick={onClick}
+      title={label}
+      subtitle=""
+      thumbnail={
+        <ObjectThumbnail
+          thumbnail={tracklet.thumbnail}
+          color={tracklet.color}
+          onClick={onThumbnailClick}
+        />
+      }
+      isMobile={isMobile}>
+      <ObjectActions objectId={tracklet.id} active={isActive} />
+    </ToolbarObjectContainer>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/ToolbarObjectContainer.tsx

@@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren, ReactNode} from 'react';
+
+const sharedStyles = stylex.create({
+  container: {
+    display: 'flex',
+    overflow: 'hidden',
+    cursor: 'pointer',
+    flexShrink: 0,
+    borderTop: 'none',
+    backgroundColor: {
+      '@media screen and (max-width: 768px)': '#000',
+    },
+    paddingHorizontal: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    paddingBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 10,
+    },
+  },
+  activeContainer: {
+    background: '#000',
+    borderRadius: 16,
+    marginHorizontal: 16,
+    padding: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    marginBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 0,
+    },
+  },
+  itemsCenter: {
+    alignItems: 'center',
+  },
+  rightColumn: {
+    marginStart: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': 0,
+    },
+    flexGrow: 1,
+    alignItems: 'center',
+  },
+});
+
+type ToolbarObjectContainerProps = PropsWithChildren<{
+  alignItems?: 'top' | 'center';
+  isActive: boolean;
+  title: string;
+  subtitle: string;
+  thumbnail: ReactNode;
+  isMobile: boolean;
+  onCancel?: () => void;
+  onClick?: () => void;
+}>;
+
+export default function ToolbarObjectContainer({
+  alignItems = 'top',
+  children,
+  isActive,
+  title,
+  subtitle,
+  thumbnail,
+  isMobile,
+  onClick,
+  onCancel,
+}: ToolbarObjectContainerProps) {
+  if (isMobile) {
+    return (
+      <div
+        onClick={onClick}
+        {...stylex.props(sharedStyles.container, sharedStyles.itemsCenter)}>
+        <div {...stylex.props(sharedStyles.rightColumn)}>{children}</div>
+      </div>
+    );
+  }
+
+  return (
+    <div
+      onClick={onClick}
+      {...stylex.props(
+        sharedStyles.container,
+        isActive && sharedStyles.activeContainer,
+        alignItems === 'center' && sharedStyles.itemsCenter,
+      )}>
+      {thumbnail}
+      <div {...stylex.props(sharedStyles.rightColumn)}>
+        <div className="text-md font-semibold ml-2">{title}</div>
+        {subtitle.length > 0 && (
+          <div className="text-sm text-gray-400 leading-5 mt-2 ml-2">
+            {subtitle}
+          </div>
+        )}
+        {children}
+      </div>
+      {onCancel != null && (
+        <div className="items-start self-stretch" onClick={onCancel}>
+          <Close size={32} />
+        </div>
+      )}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/TrackletSwimlane.tsx

@@ -0,0 +1,173 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useSelectedFrameHelper from '@/common/components/video/filmstrip/useSelectedFrameHelper';
+import {BaseTracklet, DatalessMask} from '@/common/tracker/Tracker';
+import {spacing, w} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[4],
+    width: '100%',
+  },
+  trackletNameContainer: {
+    width: w[12],
+    textAlign: 'center',
+    fontSize: '10px',
+    color: 'white',
+  },
+  swimlaneContainer: {
+    flexGrow: 1,
+    position: 'relative',
+    display: 'flex',
+    height: 12,
+    marginVertical: '0.25rem' /* 4px */,
+    '@media screen and (max-width: 768px)': {
+      marginVertical: 0,
+    },
+  },
+  swimlane: {
+    position: 'absolute',
+    left: 0,
+    top: '50%',
+    width: '100%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+    opacity: 0.4,
+  },
+  segment: {
+    position: 'absolute',
+    top: '50%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+  },
+  segmentationPoint: {
+    position: 'absolute',
+    top: '50%',
+    transform: 'translate3d(0, -50%, 0)',
+    borderRadius: '50%',
+    cursor: 'pointer',
+    width: 12,
+    height: 12,
+    '@media screen and (max-width: 768px)': {
+      width: 8,
+      height: 8,
+    },
+  },
+});
+
+type SwimlineSegment = {
+  left: number;
+  width: number;
+};
+
+type Props = {
+  tracklet: BaseTracklet;
+  onSelectFrame: (tracklet: BaseTracklet, index: number) => void;
+};
+
+function getSwimlaneSegments(masks: DatalessMask[]): SwimlineSegment[] {
+  if (masks.length === 0) {
+    return [];
+  }
+
+  const swimlineSegments: SwimlineSegment[] = [];
+  let left = -1;
+
+  for (let frameIndex = 0; frameIndex < masks.length; ++frameIndex) {
+    const isEmpty = masks?.[frameIndex]?.isEmpty ?? true;
+    if (left === -1 && !isEmpty) {
+      left = frameIndex;
+    } else if (left !== -1 && (isEmpty || frameIndex == masks.length - 1)) {
+      swimlineSegments.push({
+        left,
+        width: frameIndex - left + 1,
+      });
+      left = -1;
+    }
+  }
+
+  return swimlineSegments;
+}
+
+export default function TrackletSwimlane({tracklet, onSelectFrame}: Props) {
+  const selection = useSelectedFrameHelper();
+
+  const segments = useMemo(() => {
+    return getSwimlaneSegments(tracklet.masks);
+  }, [tracklet.masks]);
+
+  const framesWithPoints = tracklet.points.reduce<number[]>(
+    (frames, pts, frameIndex) => {
+      if (pts != null && pts.length > 0) {
+        frames.push(frameIndex);
+      }
+      return frames;
+    },
+    [],
+  );
+
+  if (selection === null) {
+    return;
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.trackletNameContainer)}>
+        Object {tracklet.id + 1}
+      </div>
+      <div {...stylex.props(styles.swimlaneContainer)}>
+        <div
+          {...stylex.props(styles.swimlane)}
+          style={{
+            backgroundColor: tracklet.color,
+          }}
+        />
+        {segments.map(segment => {
+          return (
+            <div
+              key={segment.left}
+              {...stylex.props(styles.segment)}
+              style={{
+                backgroundColor: tracklet.color,
+                left: selection.toPosition(segment.left),
+                width: selection.toPosition(segment.width),
+              }}
+            />
+          );
+        })}
+        {framesWithPoints.map(index => {
+          return (
+            <div
+              key={`frame${index}`}
+              onClick={() => {
+                onSelectFrame?.(tracklet, index);
+              }}
+              {...stylex.props(styles.segmentationPoint)}
+              style={{
+                left: Math.floor(selection.toPosition(index) - 4),
+                backgroundColor: tracklet.color,
+              }}
+            />
+          );
+        })}
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/TrackletsAnnotation.tsx

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletSwimlane from '@/common/components/annotations/TrackletSwimlane';
+import useTracklets from '@/common/components/annotations/useTracklets';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import {m, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    marginTop: m[3],
+    height: 75,
+    paddingHorizontal: spacing[4],
+    '@media screen and (max-width: 768px)': {
+      height: 25,
+    },
+  },
+});
+
+export default function TrackletsAnnotation() {
+  const video = useVideo();
+  const tracklets = useTracklets();
+
+  function handleSelectFrame(_tracklet: BaseTracklet, index: number) {
+    if (video !== null) {
+      video.frame = index;
+    }
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      {tracklets.map(tracklet => (
+        <TrackletSwimlane
+          key={tracklet.id}
+          tracklet={tracklet}
+          onSelectFrame={handleSelectFrame}
+        />
+      ))}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/annotations/useTracklets.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {trackletObjectsAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function useTracklets() {
+  return useAtomValue(trackletObjectsAtom);
+}

sam2/demo/frontend/src/common/components/button/GradientBorder.tsx

@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import stylex from '@stylexjs/stylex';
+
+import {gradients} from '@/theme/tokens.stylex';
+
+enum GradientTypes {
+  fullGradient = 'fullGradient',
+  bluePinkGradient = 'bluePinkGradient',
+}
+
+type Props = {
+  gradientType?: GradientTypes;
+  disabled?: boolean;
+  rounded?: boolean;
+  className?: string;
+} & React.DOMAttributes<HTMLDivElement>;
+
+const styles = stylex.create({
+  animationHover: {
+    ':hover': {
+      backgroundPosition: '300% 100%',
+    },
+  },
+
+  fullGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+    backgroundSize: '100% 400%',
+    transition: 'background 0.35s ease-in-out',
+  },
+
+  bluePinkGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+  },
+});
+
+export default function GradientBorder({
+  gradientType = GradientTypes.fullGradient,
+  disabled,
+  rounded = true,
+  className = '',
+  children,
+}: Props) {
+  const gradient = (name: GradientTypes) => {
+    if (name === GradientTypes.fullGradient) {
+      return styles.fullGradient;
+    } else if (name === GradientTypes.bluePinkGradient) {
+      return styles.bluePinkGradient;
+    }
+  };
+
+  return (
+    <div
+      className={`${stylex(gradient(gradientType), !disabled && styles.animationHover)} ${disabled && 'opacity-30'} ${rounded && 'rounded-full'} ${className}`}>
+      {children}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/button/PlaybackButton.tsx

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import Tooltip from '@/common/components/Tooltip';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {isPlayingAtom, streamingStateAtom, toolbarTabIndex} from '@/demo/atoms';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import {useAtomValue} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function PlaybackButton() {
+  const tabIndex = useAtomValue(toolbarTabIndex);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const video = useVideo();
+
+  const isDisabled =
+    tabIndex === OBJECT_TOOLBAR_INDEX &&
+    streamingState !== 'none' &&
+    streamingState !== 'full';
+
+  const handlePlay = useCallback(() => {
+    video?.play();
+  }, [video]);
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  const handleClick = useCallback(() => {
+    if (isDisabled) {
+      return;
+    }
+    if (isPlaying) {
+      handlePause();
+    } else {
+      handlePlay();
+    }
+  }, [isDisabled, isPlaying, handlePlay, handlePause]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleClick,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleClick]);
+
+  return (
+    <Tooltip message={`${isPlaying ? 'Pause' : 'Play'} (k)`}>
+      <button
+        disabled={isDisabled}
+        className={`group !rounded-full !w-10 !h-10 flex items-center justify-center ${getButtonStyles(isDisabled)}`}
+        onClick={handleClick}>
+        {isPlaying ? (
+          <PauseFilled size={18} />
+        ) : (
+          <PlayFilledAlt
+            size={18}
+            className={!isDisabled ? 'group-hover:text-green-500' : ''}
+          />
+        )}
+      </button>
+    </Tooltip>
+  );
+}
+
+function getButtonStyles(isDisabled: boolean): string {
+  if (isDisabled) {
+    return '!bg-gray-600 !text-graydark-700';
+  }
+  return `!text-black bg-white`;
+}

sam2/demo/frontend/src/common/components/button/PrimaryCTAButton.tsx

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+  disabled?: boolean;
+  endIcon?: ReactNode;
+} & React.DOMAttributes<HTMLButtonElement>;
+
+export default function PrimaryCTAButton({
+  children,
+  disabled,
+  endIcon,
+  ...props
+}: Props) {
+  return (
+    <GradientBorder disabled={disabled}>
+      <button
+        className={`btn ${disabled && 'btn-disabled'} !rounded-full !bg-black !text-white !border-none`}
+        {...props}>
+        {children}
+        {endIcon != null && endIcon}
+      </button>
+    </GradientBorder>
+  );
+}

sam2/demo/frontend/src/common/components/button/ResponsiveButton.tsx

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useScreenSize from '@/common/screen/useScreenSize';
+import type {ReactNode} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = ButtonProps & {startIcon: ReactNode};
+
+export default function ResponsiveButton(props: Props) {
+  const {isMobile} = useScreenSize();
+
+  return <Button {...props}>{!isMobile && props.children}</Button>;
+}

sam2/demo/frontend/src/common/components/button/TrackAndPlayButton.tsx

@@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useFunctionThrottle from '@/common/components/useFunctionThrottle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  areTrackletObjectsInitializedAtom,
+  isStreamingAtom,
+  sessionAtom,
+  streamingStateAtom,
+} from '@/demo/atoms';
+import {ChevronRight} from '@carbon/icons-react';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function TrackAndPlayButton() {
+  const video = useVideo();
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const areObjectsInitialized = useAtomValue(areTrackletObjectsInitializedAtom);
+  const setSession = useSetAtom(sessionAtom);
+  const {enqueueMessage} = useMessagesSnackbar();
+  const {isThrottled, maxThrottles, throttle} = useFunctionThrottle(250, 4);
+
+  const isTrackAndPlayDisabled =
+    streamingState === 'aborting' || streamingState === 'requesting';
+
+  useEffect(() => {
+    function onStreamingStarted() {
+      setIsStreaming(true);
+    }
+    video?.addEventListener('streamingStarted', onStreamingStarted);
+
+    function onStreamingCompleted() {
+      enqueueMessage('trackAndPlayComplete');
+      setIsStreaming(false);
+    }
+    video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+    return () => {
+      video?.removeEventListener('streamingStarted', onStreamingStarted);
+      video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+    };
+  }, [video, setIsStreaming, enqueueMessage]);
+
+  const handleTrackAndPlay = useCallback(() => {
+    if (isTrackAndPlayDisabled) {
+      return;
+    }
+    if (maxThrottles && isThrottled) {
+      enqueueMessage('trackAndPlayThrottlingWarning');
+    }
+
+    // Throttling is only applied while streaming because we should
+    // only throttle after a user has aborted inference. This way,
+    // a user can still quickly abort a stream if they notice the
+    // inferred mask is misaligned.
+    throttle(
+      () => {
+        if (!isStreaming) {
+          enqueueMessage('trackAndPlayClick');
+          video?.streamMasks();
+          setSession(previousSession =>
+            previousSession == null
+              ? previousSession
+              : {...previousSession, ranPropagation: true},
+          );
+        } else {
+          video?.abortStreamMasks();
+        }
+      },
+      {enableThrottling: isStreaming},
+    );
+  }, [
+    isTrackAndPlayDisabled,
+    isThrottled,
+    isStreaming,
+    maxThrottles,
+    video,
+    setSession,
+    enqueueMessage,
+    throttle,
+  ]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleTrackAndPlay,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleTrackAndPlay]);
+
+  return (
+    <PrimaryCTAButton
+      disabled={isThrottled || !areObjectsInitialized}
+      onClick={handleTrackAndPlay}
+      endIcon={isStreaming ? undefined : <ChevronRight size={20} />}>
+      {isStreaming ? 'Cancel Tracking' : 'Track objects'}
+    </PrimaryCTAButton>
+  );
+}

sam2/demo/frontend/src/common/components/code/InitializeLocalMonaco.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {loader} from '@monaco-editor/react';
+
+import Logger from '@/common/logger/Logger';
+import * as monaco from 'monaco-editor';
+import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker';
+import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker';
+
+self.MonacoEnvironment = {
+  getWorker(_, label) {
+    if (label === 'typescript' || label === 'javascript') {
+      return new tsWorker();
+    }
+    return new editorWorker();
+  },
+};
+
+loader.config({monaco});
+
+loader.init().then(monaco => {
+  Logger.debug('initialized monaco', monaco);
+});

sam2/demo/frontend/src/common/components/effects/BackgroundEffects.tsx

@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {backgroundEffects} from '@/common/components/effects/EffectsUtils';
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {activeBackgroundEffectAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function BackgroundEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeBackgroundEffectAtom);
+
+  return (
+    <ToolbarSection title="Background" borderBottom={false}>
+      {backgroundEffects.map(backgroundEffect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={backgroundEffect.title}
+            icon={backgroundEffect.Icon}
+            title={backgroundEffect.title}
+            isActive={activeEffect.name === backgroundEffect.effectName}
+            badge={
+              activeEffect.name === backgroundEffect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              if (activeEffect.name === backgroundEffect.effectName) {
+                setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND, {
+                  variant:
+                    (activeEffect.variant + 1) % activeEffect.numVariants,
+                });
+              } else {
+                setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND);
+              }
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}

sam2/demo/frontend/src/common/components/effects/EffectVariantBadge.tsx

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {right, top} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  variantBadge: {
+    position: 'absolute',
+    top: top[1],
+    right: right[1],
+    backgroundColor: '#280578',
+    color: '#D2D2FF',
+    fontVariantNumeric: 'tabular-nums',
+    paddingHorizontal: 4,
+    paddingVertical: 1,
+    fontSize: 9,
+    borderRadius: 6,
+    fontWeight: 'bold',
+  },
+});
+
+type Props = {
+  label: string;
+};
+
+export default function VariantBadge({label}: Props) {
+  return <div {...stylex.props(styles.variantBadge)}>{label}</div>;
+}

sam2/demo/frontend/src/common/components/effects/EffectsCarousel.tsx

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {CarouselContainerShadow} from '@/common/components/effects/EffectsCarouselShadow';
+import {DemoEffect} from '@/common/components/effects/EffectsUtils';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import type {EffectIndex} from '@/common/components/video/effects/Effects';
+import {Effects} from '@/common/components/video/effects/Effects';
+import {color, fontSize, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+type Props = {
+  label: string;
+  effects: DemoEffect[];
+  activeEffect: keyof Effects;
+  index: EffectIndex;
+};
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing[2],
+    width: '100%',
+  },
+  label: {
+    fontSize: fontSize['xs'],
+    color: '#A6ACB2',
+    textAlign: 'center',
+  },
+  carouselContainer: {
+    position: 'relative',
+    borderRadius: '8px',
+    overflow: 'hidden',
+    width: '100%',
+    height: '120px',
+    backgroundColor: color['gray-700'],
+  },
+});
+
+export default function EffectsCarousel({
+  label,
+  effects,
+  activeEffect,
+  index: effectIndex,
+}: Props) {
+  const setEffect = useVideoEffect();
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.label)}>{label}</div>
+      <div {...stylex.props(styles.carouselContainer)}>
+        <CarouselContainerShadow isTop={true} />
+        <div className="carousel carousel-vertical w-full h-full text-white">
+          <div className={`carousel-item h-6`} />
+          {effects.map(({effectName, Icon, title}, index) => {
+            const isActive = activeEffect === effectName;
+            return (
+              <div
+                key={index}
+                className={`carousel-item flex items-center h-6 gap-2 px-4`}
+                onClick={() => setEffect(effectName, effectIndex)}>
+                <Icon
+                  color={isActive ? '#FB73A5' : undefined}
+                  size={18}
+                  fontWeight={10}
+                />
+                <div
+                  className={`text-sm ${isActive ? 'text-[#FB73A5] font-bold' : 'font-medium'}`}>
+                  {title}
+                </div>
+              </div>
+            );
+          })}
+          <div className={`carousel-item h-6`} />
+        </div>
+        <CarouselContainerShadow isTop={false} />
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/effects/EffectsCarouselShadow.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    width: '100%',
+    height: spacing[8],
+    pointerEvents: 'none',
+  },
+});
+
+type CarouselContainerShadowProps = {
+  isTop: boolean;
+};
+
+const edgeColor = 'rgba(55, 62, 65, 1)';
+const transitionColor = 'rgba(55, 62, 65, 0.2)';
+
+export function CarouselContainerShadow({isTop}: CarouselContainerShadowProps) {
+  return (
+    <div
+      {...stylex.props(styles.container)}
+      style={{
+        background: `linear-gradient(${isTop ? `${edgeColor}, ${transitionColor}` : `${transitionColor}, ${edgeColor}`})`,
+        top: isTop ? 0 : undefined,
+        bottom: isTop ? undefined : 0,
+      }}
+    />
+  );
+}

sam2/demo/frontend/src/common/components/effects/EffectsToolbar.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BackgroundEffects from '@/common/components/effects/BackgroundEffects';
+import EffectsToolbarBottomActions from '@/common/components/effects/EffectsToolbarBottomActions';
+import EffectsToolbarHeader from '@/common/components/effects/EffectsToolbarHeader';
+import HighlightEffects from '@/common/components/effects/HighlightEffects';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {useEffect, useRef} from 'react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function EffectsToolbar({onTabChange}: Props) {
+  const isEffectsMessageShown = useRef(false);
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useEffect(() => {
+    if (!isEffectsMessageShown.current) {
+      isEffectsMessageShown.current = true;
+      enqueueMessage('effectsMessage');
+    }
+  }, [enqueueMessage]);
+
+  return (
+    <div className="flex flex-col h-full">
+      <EffectsToolbarHeader />
+      <div className="grow overflow-y-auto">
+        <HighlightEffects />
+        <BackgroundEffects />
+      </div>
+      <EffectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/effects/EffectsToolbarBottomActions.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import RestartSessionButton from '@/common/components/session/RestartSessionButton';
+import ToolbarBottomActionsWrapper from '@/common/components/toolbar/ToolbarBottomActionsWrapper';
+import {
+  MORE_OPTIONS_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {ChevronRight} from '@carbon/icons-react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function EffectsToolbarBottomActions({onTabChange}: Props) {
+  function handleSwitchToMoreOptionsTab() {
+    onTabChange(MORE_OPTIONS_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <RestartSessionButton
+        onRestartSession={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+      <PrimaryCTAButton
+        onClick={handleSwitchToMoreOptionsTab}
+        endIcon={<ChevronRight />}>
+        Next
+      </PrimaryCTAButton>
+    </ToolbarBottomActionsWrapper>
+  );
+}

sam2/demo/frontend/src/common/components/effects/EffectsToolbarHeader.tsx

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {
+  EffectIndex,
+  effectPresets,
+} from '@/common/components/video/effects/Effects';
+import {BLUE_PINK_FILL} from '@/theme/gradientStyle';
+import {MagicWandFilled} from '@carbon/icons-react';
+import {useCallback, useRef} from 'react';
+import {Button} from 'react-daisyui';
+
+export default function EffectsToolbarHeader() {
+  const preset = useRef(0);
+  const setEffect = useVideoEffect();
+
+  const handleTogglePreset = useCallback(() => {
+    preset.current++;
+    const [background, highlight] =
+      effectPresets[preset.current % effectPresets.length];
+    setEffect(background.name, EffectIndex.BACKGROUND, {
+      variant: background.variant,
+    });
+    setEffect(highlight.name, EffectIndex.HIGHLIGHT, {
+      variant: highlight.variant,
+    });
+  }, [setEffect]);
+
+  return (
+    <ToolbarHeaderWrapper
+      title="Add effects"
+      description="Apply visual effects to your selected objects and the background. Keeping clicking the same effect for different variations."
+      bottomSection={
+        <div className="flex mt-1">
+          <Button
+            color="ghost"
+            size="md"
+            className={`font-medium bg-black !rounded-full hover:!bg-gradient-to-br ${BLUE_PINK_FILL} border-none`}
+            endIcon={<MagicWandFilled size={20} className="text-white " />}
+            onClick={handleTogglePreset}>
+            Surprise Me
+          </Button>
+        </div>
+      }
+      className="pb-4"
+    />
+  );
+}

sam2/demo/frontend/src/common/components/effects/EffectsUtils.ts

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Effects} from '@/common/components/video/effects/Effects';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {
+  AppleDash,
+  Asterisk,
+  Barcode,
+  CenterCircle,
+  ColorPalette,
+  ColorSwitch,
+  Development,
+  Erase,
+  FaceWink,
+  Humidity,
+  Image,
+  Overlay,
+  TextFont,
+} from '@carbon/icons-react';
+
+export type DemoEffect = {
+  title: string;
+  Icon: CarbonIconType;
+  effectName: keyof Effects;
+};
+
+export const backgroundEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Original'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseBackground'},
+  {
+    title: 'Gradient',
+    Icon: ColorPalette,
+    effectName: 'Gradient',
+  },
+  {
+    title: 'Pixelate',
+    Icon: Development,
+    effectName: 'Pixelate',
+  },
+  {title: 'Desaturate', Icon: ColorSwitch, effectName: 'Desaturate'},
+  {title: 'Text', Icon: TextFont, effectName: 'BackgroundText'},
+  {title: 'Blur', Icon: Humidity, effectName: 'BackgroundBlur'},
+  {title: 'Outline', Icon: AppleDash, effectName: 'Sobel'},
+];
+
+export const highlightEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Cutout'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseForeground'},
+  {title: 'Gradient', Icon: ColorPalette, effectName: 'VibrantMask'},
+  {title: 'Pixelate', Icon: Development, effectName: 'PixelateMask'},
+  {
+    title: 'Overlay',
+    Icon: Overlay,
+    effectName: 'Overlay',
+  },
+  {title: 'Emoji', Icon: FaceWink, effectName: 'Replace'},
+  {title: 'Burst', Icon: Asterisk, effectName: 'Burst'},
+  {title: 'Spotlight', Icon: CenterCircle, effectName: 'Scope'},
+];
+
+export const moreEffects: DemoEffect[] = [
+  {title: 'Noisy', Icon: Barcode, effectName: 'NoisyMask'},
+];

sam2/demo/frontend/src/common/components/effects/HighlightEffects.tsx

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {
+  activeHighlightEffectAtom,
+  activeHighlightEffectGroupAtom,
+} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function HighlightEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeHighlightEffectAtom);
+  const activeEffectsGroup = useAtomValue(activeHighlightEffectGroupAtom);
+
+  return (
+    <ToolbarSection title="Selected Objects" borderBottom={true}>
+      {activeEffectsGroup.map(highlightEffect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={highlightEffect.title}
+            icon={highlightEffect.Icon}
+            title={highlightEffect.title}
+            isActive={activeEffect.name === highlightEffect.effectName}
+            badge={
+              activeEffect.name === highlightEffect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              if (activeEffect.name === highlightEffect.effectName) {
+                setEffect(highlightEffect.effectName, EffectIndex.HIGHLIGHT, {
+                  variant:
+                    (activeEffect.variant + 1) % activeEffect.numVariants,
+                });
+              } else {
+                setEffect(highlightEffect.effectName, EffectIndex.HIGHLIGHT);
+              }
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}

sam2/demo/frontend/src/common/components/effects/MobileEffectsToolbar.tsx

@@ -0,0 +1,115 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import EffectsCarousel from '@/common/components/effects/EffectsCarousel';
+import {backgroundEffects} from '@/common/components/effects/EffectsUtils';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {
+  EffectIndex,
+  effectPresets,
+} from '@/common/components/video/effects/Effects';
+import {ListBoxes, MagicWand, MagicWandFilled} from '@carbon/icons-react';
+import {useCallback, useRef, useState} from 'react';
+import {Button} from 'react-daisyui';
+
+import EffectsToolbarBottomActions from '@/common/components/effects/EffectsToolbarBottomActions';
+import ToolbarProgressChip from '@/common/components/toolbar/ToolbarProgressChip';
+import {
+  activeBackgroundEffectAtom,
+  activeHighlightEffectAtom,
+  activeHighlightEffectGroupAtom,
+} from '@/demo/atoms';
+import {BLUE_PINK_FILL} from '@/theme/gradientStyle';
+import {useAtomValue} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileEffectsToolbar({onTabChange}: Props) {
+  const preset = useRef(0);
+  const setEffect = useVideoEffect();
+  const [showEffectsCarousels, setShowEffectsCarousels] = useState<boolean>();
+
+  const activeBackground = useAtomValue(activeBackgroundEffectAtom);
+  const activeHighlight = useAtomValue(activeHighlightEffectAtom);
+  const activeHighlightEffectsGroup = useAtomValue(
+    activeHighlightEffectGroupAtom,
+  );
+
+  const handleTogglePreset = useCallback(() => {
+    preset.current++;
+    const [background, highlight] =
+      effectPresets[preset.current % effectPresets.length];
+    setEffect(background.name, EffectIndex.BACKGROUND, {
+      variant: background.variant,
+    });
+    setEffect(highlight.name, EffectIndex.HIGHLIGHT, {
+      variant: highlight.variant,
+    });
+  }, [setEffect]);
+
+  return (
+    <div className="w-full">
+      {showEffectsCarousels ? (
+        <div className="flex gap-2 px-2 py-4 items-center p-6">
+          <Button
+            color="ghost"
+            className="mt-6 !px-2 !text-[#FB73A5]"
+            startIcon={<MagicWand size={20} />}
+            onClick={handleTogglePreset}
+          />
+          <EffectsCarousel
+            label="Highlights"
+            effects={activeHighlightEffectsGroup}
+            activeEffect={activeHighlight.name}
+            index={1}
+          />
+          <EffectsCarousel
+            label="Background"
+            effects={backgroundEffects}
+            activeEffect={activeBackground.name}
+            index={0}
+          />
+        </div>
+      ) : (
+        <div className="flex flex-col gap-6 p-6">
+          <div className="text-sm text-white">
+            <ToolbarProgressChip />
+            Apply visual effects to your selected objects and the background.
+          </div>
+          <div className="grid grid-cols-2 gap-2">
+            <Button
+              color="ghost"
+              endIcon={<MagicWandFilled size={20} />}
+              className={`font-bold bg-black !rounded-full !bg-gradient-to-br ${BLUE_PINK_FILL} border-none text-white`}
+              onClick={handleTogglePreset}>
+              Surprise Me
+            </Button>
+            <Button
+              color="ghost"
+              className={`font-bold bg-black !rounded-full border-none text-white`}
+              startIcon={<ListBoxes size={20} />}
+              onClick={() => setShowEffectsCarousels(true)}>
+              More effects
+            </Button>
+          </div>
+        </div>
+      )}
+
+      <EffectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/effects/MoreFunEffects.tsx

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {moreEffects} from '@/common/components/effects/EffectsUtils';
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {activeHighlightEffectAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function MoreFunEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeHighlightEffectAtom);
+
+  return (
+    <ToolbarSection title="Selected Objects" borderBottom={true}>
+      {moreEffects.map(effect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={effect.title}
+            icon={effect.Icon}
+            title={effect.title}
+            isActive={activeEffect.name === effect.effectName}
+            badge={
+              activeEffect.name === effect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              setEffect(effect.effectName, EffectIndex.HIGHLIGHT);
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}

sam2/demo/frontend/src/common/components/gallery/ChangeVideoModal.tsx

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import DemoVideoGalleryModal from '@/common/components/gallery/DemoVideoGalleryModal';
+import useVideo from '@/common/components/video/editor/useVideo';
+import Logger from '@/common/logger/Logger';
+import {isStreamingAtom, uploadingStateAtom, VideoData} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {ComponentType, useCallback} from 'react';
+import {useNavigate} from 'react-router-dom';
+
+type Props = {
+  videoGalleryModalTrigger?: ComponentType<VideoGalleryTriggerProps>;
+  showUploadInGallery?: boolean;
+  onChangeVideo?: () => void;
+};
+
+export default function ChangeVideoModal({
+  videoGalleryModalTrigger: VideoGalleryModalTriggerComponent,
+  showUploadInGallery = true,
+  onChangeVideo,
+}: Props) {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const video = useVideo();
+  const navigate = useNavigate();
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  function handlePauseOrAbortVideo() {
+    if (isStreaming) {
+      video?.abortStreamMasks();
+    } else {
+      handlePause();
+    }
+  }
+
+  function handleSwitchVideos(video: VideoData) {
+    // Retain any search parameter
+    navigate(
+      {
+        pathname: location.pathname,
+        search: location.search,
+      },
+      {
+        state: {
+          video,
+        },
+      },
+    );
+    onChangeVideo?.();
+  }
+
+  function handleUploadVideoError(error: Error) {
+    setUploadingState('error');
+    Logger.error(error);
+  }
+
+  return (
+    <DemoVideoGalleryModal
+      trigger={VideoGalleryModalTriggerComponent}
+      showUploadInGallery={showUploadInGallery}
+      onOpen={handlePauseOrAbortVideo}
+      onSelect={handleSwitchVideos}
+      onUploadVideoError={handleUploadVideoError}
+    />
+  );
+}

sam2/demo/frontend/src/common/components/gallery/DefaultVideoGalleryModalTrigger.tsx

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ResponsiveButton from '@/common/components/button/ResponsiveButton';
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import {ImageCopy} from '@carbon/icons-react';
+
+export default function DefaultVideoGalleryModalTrigger({
+  onClick,
+}: VideoGalleryTriggerProps) {
+  return (
+    <ResponsiveButton
+      color="ghost"
+      className="hover:!bg-black"
+      startIcon={<ImageCopy size={20} />}
+      onClick={onClick}>
+      Change video
+    </ResponsiveButton>
+  );
+}

sam2/demo/frontend/src/common/components/gallery/DemoVideoGallery.tsx

@@ -0,0 +1,209 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {DemoVideoGalleryQuery} from '@/common/components/gallery/__generated__/DemoVideoGalleryQuery.graphql';
+import VideoGalleryUploadVideo from '@/common/components/gallery/VideoGalleryUploadPhoto';
+import VideoPhoto from '@/common/components/gallery/VideoPhoto';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {VideoData} from '@/demo/atoms';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {fontSize, fontWeight, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+import PhotoAlbum, {Photo, RenderPhotoProps} from 'react-photo-album';
+import {graphql, useLazyLoadQuery} from 'react-relay';
+import {useLocation, useNavigate} from 'react-router-dom';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    marginHorizontal: spacing[1],
+    height: '100%',
+    lineHeight: 1.2,
+    paddingTop: spacing[8],
+  },
+  headerContainer: {
+    marginBottom: spacing[8],
+    fontWeight: fontWeight['medium'],
+    fontSize: fontSize['2xl'],
+    '@media screen and (max-width: 768px)': {
+      marginTop: spacing[0],
+      marginBottom: spacing[8],
+      marginHorizontal: spacing[4],
+      fontSize: fontSize['xl'],
+    },
+  },
+  albumContainer: {
+    flex: '1 1 0%',
+    width: '100%',
+    overflowY: 'auto',
+  },
+});
+
+type Props = {
+  showUploadInGallery?: boolean;
+  onSelect?: (video: VideoPhotoData) => void;
+  onUpload: (video: VideoData) => void;
+  onUploadStart?: () => void;
+  onUploadError?: (error: Error) => void;
+};
+
+type VideoPhotoData = Photo &
+  VideoData & {
+    poster: string;
+    isUploadOption: boolean;
+  };
+
+export default function DemoVideoGallery({
+  showUploadInGallery = false,
+  onSelect,
+  onUpload,
+  onUploadStart,
+  onUploadError,
+}: Props) {
+  const navigate = useNavigate();
+  const location = useLocation();
+  const {isMobile: isMobileScreenSize} = useScreenSize();
+
+  const data = useLazyLoadQuery<DemoVideoGalleryQuery>(
+    graphql`
+      query DemoVideoGalleryQuery {
+        videos {
+          edges {
+            node {
+              id
+              path
+              posterPath
+              url
+              posterUrl
+              height
+              width
+              posterUrl
+            }
+          }
+        }
+      }
+    `,
+    {},
+  );
+
+  const allVideos: VideoPhotoData[] = useMemo(() => {
+    return data.videos.edges.map(video => {
+      return {
+        src: video.node.url,
+        path: video.node.path,
+        poster: video.node.posterPath,
+        posterPath: video.node.posterPath,
+        url: video.node.url,
+        posterUrl: video.node.posterUrl,
+        width: video.node.width,
+        height: video.node.height,
+        isUploadOption: false,
+      } as VideoPhotoData;
+    });
+  }, [data.videos.edges]);
+
+  const shareableVideos: VideoPhotoData[] = useMemo(() => {
+    const filteredVideos = [...allVideos];
+
+    if (showUploadInGallery) {
+      const uploadOption = {
+        src: '',
+        width: 1280,
+        height: 720,
+        poster: '',
+        isUploadOption: true,
+      } as VideoPhotoData;
+      filteredVideos.unshift(uploadOption);
+    }
+
+    return filteredVideos;
+  }, [allVideos, showUploadInGallery]);
+
+  const renderPhoto = ({
+    photo: video,
+    imageProps,
+  }: RenderPhotoProps<VideoPhotoData>) => {
+    const {style} = imageProps;
+    const {url, posterUrl} = video;
+
+    return video.isUploadOption ? (
+      <VideoGalleryUploadVideo
+        style={style}
+        onUpload={handleUploadVideo}
+        onUploadError={onUploadError}
+        onUploadStart={onUploadStart}
+      />
+    ) : (
+      <VideoPhoto
+        src={url}
+        poster={posterUrl}
+        style={style}
+        onClick={() => {
+          navigate(location.pathname, {
+            state: {
+              video,
+            },
+          });
+          onSelect?.(video);
+        }}
+      />
+    );
+  };
+
+  function handleUploadVideo(video: VideoData) {
+    navigate(location.pathname, {
+      state: {
+        video,
+      },
+    });
+    onUpload?.(video);
+  }
+
+  const descriptionStyle = 'text-sm md:text-base text-gray-400 leading-snug';
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.albumContainer)}>
+        <div className="pt-0 md:px-16 md:pt-8 md:pb-8">
+          <div {...stylex.props(styles.headerContainer)}>
+            <h3 className="mb-2">
+              Select a video to try{' '}
+              <span className="hidden md:inline">
+                with the {DEMO_SHORT_NAME}
+              </span>
+            </h3>
+            <p className={descriptionStyle}>
+              You’ll be able to download what you make.
+            </p>
+          </div>
+
+          <PhotoAlbum<VideoPhotoData>
+            layout="rows"
+            photos={shareableVideos}
+            targetRowHeight={isMobileScreenSize ? 120 : 200}
+            rowConstraints={{
+              singleRowMaxHeight: isMobileScreenSize ? 120 : 240,
+              maxPhotos: 3,
+            }}
+            renderPhoto={renderPhoto}
+            spacing={4}
+          />
+        </div>
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/gallery/DemoVideoGalleryModal.tsx

@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import DefaultVideoGalleryModalTrigger from '@/common/components/gallery/DefaultVideoGalleryModalTrigger';
+import {
+  frameIndexAtom,
+  sessionAtom,
+  uploadingStateAtom,
+  VideoData,
+} from '@/demo/atoms';
+import {spacing} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {useSetAtom} from 'jotai';
+import {ComponentType, useCallback, useRef} from 'react';
+import {Modal} from 'react-daisyui';
+import DemoVideoGallery from './DemoVideoGallery';
+
+const styles = stylex.create({
+  container: {
+    position: 'relative',
+    minWidth: '85vw',
+    minHeight: '85vh',
+    overflow: 'hidden',
+    color: '#fff',
+    boxShadow: '0 0 100px 50px #000',
+    borderRadius: 16,
+    border: '2px solid transparent',
+    background:
+      'linear-gradient(#1A1C1F, #1A1C1F) padding-box, linear-gradient(to right bottom, #FB73A5,#595FEF,#94EAE2,#FCCB6B) border-box',
+  },
+  closeButton: {
+    position: 'absolute',
+    top: 0,
+    right: 0,
+    padding: spacing[3],
+    zIndex: 10,
+    cursor: 'pointer',
+    ':hover': {
+      opacity: 0.7,
+    },
+  },
+  galleryContainer: {
+    position: 'absolute',
+    top: spacing[4],
+    left: 0,
+    right: 0,
+    bottom: 0,
+    overflowY: 'auto',
+  },
+});
+
+export type VideoGalleryTriggerProps = {
+  onClick: () => void;
+};
+
+type Props = {
+  trigger?: ComponentType<VideoGalleryTriggerProps>;
+  showUploadInGallery?: boolean;
+  onOpen?: () => void;
+  onSelect?: (video: VideoData, isUpload?: boolean) => void;
+  onUploadVideoError?: (error: Error) => void;
+};
+
+export default function DemoVideoGalleryModal({
+  trigger: VideoGalleryModalTrigger = DefaultVideoGalleryModalTrigger,
+  showUploadInGallery = false,
+  onOpen,
+  onSelect,
+  onUploadVideoError,
+}: Props) {
+  const modalRef = useRef<HTMLDialogElement | null>(null);
+
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const setSession = useSetAtom(sessionAtom);
+
+  function openModal() {
+    const modal = modalRef.current;
+    if (modal != null) {
+      modal.style.display = 'grid';
+      modal.showModal();
+    }
+  }
+
+  function closeModal() {
+    const modal = modalRef.current;
+    if (modal != null) {
+      modal.close();
+      modal.style.display = 'none';
+    }
+  }
+
+  const handleSelect = useCallback(
+    async (video: VideoData, isUpload?: boolean) => {
+      closeModal();
+      setFrameIndex(0);
+      onSelect?.(video, isUpload);
+      setUploadingState('default');
+      setSession(null);
+    },
+    [setFrameIndex, onSelect, setUploadingState, setSession],
+  );
+
+  function handleUploadVideoStart() {
+    setUploadingState('uploading');
+    closeModal();
+  }
+
+  function handleOpenVideoGalleryModal() {
+    onOpen?.();
+    openModal();
+  }
+
+  return (
+    <>
+      <VideoGalleryModalTrigger onClick={handleOpenVideoGalleryModal} />
+      <Modal ref={modalRef} {...stylex.props(styles.container)}>
+        <div onClick={closeModal} {...stylex.props(styles.closeButton)}>
+          <Close size={28} />
+        </div>
+        <Modal.Body>
+          <div {...stylex.props(styles.galleryContainer)}>
+            <DemoVideoGallery
+              showUploadInGallery={showUploadInGallery}
+              onSelect={video => handleSelect(video)}
+              onUpload={video => handleSelect(video, true)}
+              onUploadStart={handleUploadVideoStart}
+              onUploadError={onUploadVideoError}
+            />
+          </div>
+        </Modal.Body>
+      </Modal>
+    </>
+  );
+}

sam2/demo/frontend/src/common/components/gallery/VideoGalleryUploadPhoto.tsx

@@ -0,0 +1,102 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useUploadVideo from '@/common/components/gallery/useUploadVideo';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {VideoData} from '@/demo/atoms';
+import {MAX_UPLOAD_FILE_SIZE} from '@/demo/DemoConfig';
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+import {RetryFailed, Upload} from '@carbon/icons-react';
+import {CSSProperties, ReactNode} from 'react';
+import {Loading} from 'react-daisyui';
+
+type Props = {
+  style: CSSProperties;
+  onUpload: (video: VideoData) => void;
+  onUploadStart?: () => void;
+  onUploadError?: (error: Error) => void;
+};
+
+export default function VideoGalleryUploadVideo({
+  style,
+  onUpload,
+  onUploadStart,
+  onUploadError,
+}: Props) {
+  const {getRootProps, getInputProps, isUploading, error} = useUploadVideo({
+    onUpload,
+    onUploadStart,
+    onUploadError,
+  });
+  const {isMobile} = useScreenSize();
+
+  return (
+    <div className={`cursor-pointer ${BLUE_PINK_FILL_BR}`} style={style}>
+      <span {...getRootProps()}>
+        <input {...getInputProps()} />
+        <div className="relative w-full h-full">
+          <div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2">
+            {isUploading && (
+              <IconWrapper
+                icon={
+                  <Loading
+                    size={isMobile ? 'md' : 'lg'}
+                    className="text-white"
+                  />
+                }
+                title="Uploading ..."
+              />
+            )}
+            {error !== null && (
+              <IconWrapper
+                icon={<RetryFailed color="white" size={isMobile ? 24 : 32} />}
+                title={error}
+              />
+            )}
+            {!isUploading && error === null && (
+              <IconWrapper
+                icon={<Upload color="white" size={isMobile ? 24 : 32} />}
+                title={
+                  <>
+                    Upload{' '}
+                    <div className="text-xs opacity-70">
+                      Max {MAX_UPLOAD_FILE_SIZE}
+                    </div>
+                  </>
+                }
+              />
+            )}
+          </div>
+        </div>
+      </span>
+    </div>
+  );
+}
+
+type IconWrapperProps = {
+  icon: ReactNode;
+  title: ReactNode | string;
+};
+
+function IconWrapper({icon, title}: IconWrapperProps) {
+  return (
+    <>
+      <div className="flex justify-center">{icon}</div>
+      <div className="mt-1 text-sm md:text-lg text-white font-medium text-center leading-tight">
+        {title}
+      </div>
+    </>
+  );
+}

sam2/demo/frontend/src/common/components/gallery/VideoPhoto.tsx

@@ -0,0 +1,112 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Logger from '@/common/logger/Logger';
+import stylex from '@stylexjs/stylex';
+import {
+  CSSProperties,
+  MouseEventHandler,
+  useCallback,
+  useEffect,
+  useRef,
+} from 'react';
+
+const styles = stylex.create({
+  background: {
+    backgroundRepeat: 'no-repeat',
+    backgroundSize: 'cover',
+    backgroundPosition: 'center',
+    cursor: 'pointer',
+  },
+  video: {
+    width: '100%',
+    height: '100%',
+  },
+});
+
+type Props = {
+  onClick: MouseEventHandler<HTMLVideoElement> | undefined;
+  src: string;
+  poster: string;
+  style: CSSProperties;
+};
+
+export default function VideoPhoto({src, poster, style, onClick}: Props) {
+  const videoRef = useRef<HTMLVideoElement | null>(null);
+  const playPromiseRef = useRef<Promise<void> | null>(null);
+
+  const play = useCallback(() => {
+    const video = videoRef.current;
+    // Only play video if it is not already playing
+    if (video != null && video.paused) {
+      // This quirky way of handling video play/pause in the browser is needed
+      // due to the async nature of the video play API:
+      // https://developer.chrome.com/blog/play-request-was-interrupted/
+      const playPromise = video.play();
+      playPromise.catch(error => {
+        Logger.error('Failed to play video', error);
+      });
+      playPromiseRef.current = playPromise;
+    }
+  }, []);
+
+  const pause = useCallback(() => {
+    // Only pause video if it is playing
+    const playPromise = playPromiseRef.current;
+    if (playPromise != null) {
+      playPromise
+        .then(() => {
+          videoRef.current?.pause();
+        })
+        .catch(error => {
+          Logger.error('Failed to pause video', error);
+        })
+        .finally(() => {
+          playPromiseRef.current = null;
+        });
+    }
+  }, []);
+
+  useEffect(() => {
+    return () => {
+      pause();
+    };
+  }, [pause]);
+
+  return (
+    <div
+      style={{
+        ...style,
+        backgroundImage: `url(${poster})`,
+      }}
+      {...stylex.props(styles.background)}>
+      <video
+        ref={videoRef}
+        {...stylex.props(styles.video)}
+        preload="none"
+        playsInline
+        loop
+        muted
+        title="Gallery Video"
+        poster={poster}
+        onMouseEnter={play}
+        onMouseLeave={pause}
+        onClick={onClick}>
+        <source src={src} type="video/mp4" />
+        Sorry, your browser does not support embedded videos.
+      </video>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/gallery/__generated__/DemoVideoGalleryModalQuery.graphql.ts

@@ -0,0 +1,303 @@
+/**
+ * @generated SignedSource<<db7e183e1996cf656749b4e33c2424e6>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Query } from 'relay-runtime';
+import { FragmentRefs } from "relay-runtime";
+export type DemoVideoGalleryModalQuery$variables = Record<PropertyKey, never>;
+export type DemoVideoGalleryModalQuery$data = {
+  readonly " $fragmentSpreads": FragmentRefs<"DatasetsDropdown_datasets" | "VideoGallery_videos">;
+};
+export type DemoVideoGalleryModalQuery = {
+  response: DemoVideoGalleryModalQuery$data;
+  variables: DemoVideoGalleryModalQuery$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "alias": null,
+    "args": null,
+    "kind": "ScalarField",
+    "name": "name",
+    "storageKey": null
+  }
+],
+v1 = [
+  {
+    "kind": "Literal",
+    "name": "after",
+    "value": ""
+  },
+  {
+    "kind": "Literal",
+    "name": "first",
+    "value": 20
+  }
+],
+v2 = {
+  "alias": null,
+  "args": null,
+  "kind": "ScalarField",
+  "name": "__typename",
+  "storageKey": null
+};
+return {
+  "fragment": {
+    "argumentDefinitions": [],
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "DemoVideoGalleryModalQuery",
+    "selections": [
+      {
+        "args": null,
+        "kind": "FragmentSpread",
+        "name": "DatasetsDropdown_datasets"
+      },
+      {
+        "args": null,
+        "kind": "FragmentSpread",
+        "name": "VideoGallery_videos"
+      }
+    ],
+    "type": "Query",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": [],
+    "kind": "Operation",
+    "name": "DemoVideoGalleryModalQuery",
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "DatasetConnection",
+        "kind": "LinkedField",
+        "name": "datasets",
+        "plural": false,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "DatasetEdge",
+            "kind": "LinkedField",
+            "name": "edges",
+            "plural": true,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "concreteType": "Dataset",
+                "kind": "LinkedField",
+                "name": "node",
+                "plural": false,
+                "selections": (v0/*: any*/),
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": (v1/*: any*/),
+        "concreteType": "VideoConnection",
+        "kind": "LinkedField",
+        "name": "videos",
+        "plural": false,
+        "selections": [
+          (v2/*: any*/),
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "PageInfo",
+            "kind": "LinkedField",
+            "name": "pageInfo",
+            "plural": false,
+            "selections": [
+              (v2/*: any*/),
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "hasPreviousPage",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "hasNextPage",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "startCursor",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "endCursor",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          },
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "VideoEdge",
+            "kind": "LinkedField",
+            "name": "edges",
+            "plural": true,
+            "selections": [
+              (v2/*: any*/),
+              {
+                "alias": null,
+                "args": null,
+                "concreteType": "Video",
+                "kind": "LinkedField",
+                "name": "node",
+                "plural": false,
+                "selections": [
+                  (v2/*: any*/),
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "id",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "path",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "posterPath",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "url",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "posterUrl",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "width",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "kind": "ScalarField",
+                    "name": "height",
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "concreteType": "Dataset",
+                    "kind": "LinkedField",
+                    "name": "dataset",
+                    "plural": false,
+                    "selections": (v0/*: any*/),
+                    "storageKey": null
+                  },
+                  {
+                    "alias": null,
+                    "args": null,
+                    "concreteType": "VideoPermissions",
+                    "kind": "LinkedField",
+                    "name": "permissions",
+                    "plural": false,
+                    "selections": [
+                      {
+                        "alias": null,
+                        "args": null,
+                        "kind": "ScalarField",
+                        "name": "canShare",
+                        "storageKey": null
+                      },
+                      {
+                        "alias": null,
+                        "args": null,
+                        "kind": "ScalarField",
+                        "name": "canDownload",
+                        "storageKey": null
+                      }
+                    ],
+                    "storageKey": null
+                  }
+                ],
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "cursor",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": "videos(after:\"\",first:20)"
+      },
+      {
+        "alias": null,
+        "args": (v1/*: any*/),
+        "filters": [
+          "datasetName"
+        ],
+        "handle": "connection",
+        "key": "VideoGallery_videos",
+        "kind": "LinkedHandle",
+        "name": "videos"
+      }
+    ]
+  },
+  "params": {
+    "cacheID": "e0bccf553377682e6bc283c2ce53bee5",
+    "id": null,
+    "metadata": {},
+    "name": "DemoVideoGalleryModalQuery",
+    "operationKind": "query",
+    "text": "query DemoVideoGalleryModalQuery {\n  ...DatasetsDropdown_datasets\n  ...VideoGallery_videos\n}\n\nfragment DatasetsDropdown_datasets on Query {\n  datasets {\n    edges {\n      node {\n        name\n      }\n    }\n  }\n}\n\nfragment VideoGallery_videos on Query {\n  videos(first: 20, after: \"\") {\n    __typename\n    pageInfo {\n      __typename\n      hasPreviousPage\n      hasNextPage\n      startCursor\n      endCursor\n    }\n    edges {\n      __typename\n      node {\n        __typename\n        id\n        path\n        posterPath\n        url\n        posterUrl\n        width\n        height\n        dataset {\n          name\n        }\n        permissions {\n          canShare\n          canDownload\n        }\n      }\n      cursor\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "d09e34e2b9f2e25c2d564106de5f9c89";
+
+export default node;

sam2/demo/frontend/src/common/components/gallery/__generated__/DemoVideoGalleryQuery.graphql.ts

@@ -0,0 +1,148 @@
+/**
+ * @generated SignedSource<<20d31a82b5f3b251b0e42b4f0e3522b8>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Query } from 'relay-runtime';
+export type DemoVideoGalleryQuery$variables = Record<PropertyKey, never>;
+export type DemoVideoGalleryQuery$data = {
+  readonly videos: {
+    readonly edges: ReadonlyArray<{
+      readonly node: {
+        readonly height: number;
+        readonly id: any;
+        readonly path: string;
+        readonly posterPath: string | null | undefined;
+        readonly posterUrl: string;
+        readonly url: string;
+        readonly width: number;
+      };
+    }>;
+  };
+};
+export type DemoVideoGalleryQuery = {
+  response: DemoVideoGalleryQuery$data;
+  variables: DemoVideoGalleryQuery$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "alias": null,
+    "args": null,
+    "concreteType": "VideoConnection",
+    "kind": "LinkedField",
+    "name": "videos",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "VideoEdge",
+        "kind": "LinkedField",
+        "name": "edges",
+        "plural": true,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "Video",
+            "kind": "LinkedField",
+            "name": "node",
+            "plural": false,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "id",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "path",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "posterPath",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "url",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "posterUrl",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "height",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "width",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": [],
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "DemoVideoGalleryQuery",
+    "selections": (v0/*: any*/),
+    "type": "Query",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": [],
+    "kind": "Operation",
+    "name": "DemoVideoGalleryQuery",
+    "selections": (v0/*: any*/)
+  },
+  "params": {
+    "cacheID": "4dae74153a5528f2631b59dfb0adb021",
+    "id": null,
+    "metadata": {},
+    "name": "DemoVideoGalleryQuery",
+    "operationKind": "query",
+    "text": "query DemoVideoGalleryQuery {\n  videos {\n    edges {\n      node {\n        id\n        path\n        posterPath\n        url\n        posterUrl\n        height\n        width\n      }\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "d22ac5e58f6e4eb696651be49b410e4e";
+
+export default node;

sam2/demo/frontend/src/common/components/gallery/__generated__/useUploadVideoMutation.graphql.ts

@@ -0,0 +1,137 @@
+/**
+ * @generated SignedSource<<76014dced98d6c8989e7322712e38963>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type useUploadVideoMutation$variables = {
+  file: any;
+};
+export type useUploadVideoMutation$data = {
+  readonly uploadVideo: {
+    readonly height: number;
+    readonly id: any;
+    readonly path: string;
+    readonly posterPath: string | null | undefined;
+    readonly posterUrl: string;
+    readonly url: string;
+    readonly width: number;
+  };
+};
+export type useUploadVideoMutation = {
+  response: useUploadVideoMutation$data;
+  variables: useUploadVideoMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "file"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "file",
+        "variableName": "file"
+      }
+    ],
+    "concreteType": "Video",
+    "kind": "LinkedField",
+    "name": "uploadVideo",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "id",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "height",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "width",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "url",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "path",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "posterPath",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "posterUrl",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "useUploadVideoMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "useUploadVideoMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "dcbaf1bf411627fdb9dfbb827592cfc0",
+    "id": null,
+    "metadata": {},
+    "name": "useUploadVideoMutation",
+    "operationKind": "mutation",
+    "text": "mutation useUploadVideoMutation(\n  $file: Upload!\n) {\n  uploadVideo(file: $file) {\n    id\n    height\n    width\n    url\n    path\n    posterPath\n    posterUrl\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "710e462504d76597af8695b7fc70b4cf";
+
+export default node;

sam2/demo/frontend/src/common/components/gallery/useUploadVideo.ts

@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useUploadVideoMutation} from '@/common/components/gallery/__generated__/useUploadVideoMutation.graphql';
+import Logger from '@/common/logger/Logger';
+import {VideoData} from '@/demo/atoms';
+import {useState} from 'react';
+import {FileRejection, FileWithPath, useDropzone} from 'react-dropzone';
+import {graphql, useMutation} from 'react-relay';
+
+const ACCEPT_VIDEOS = {
+  'video/mp4': ['.mp4'],
+  'video/quicktime': ['.mov'],
+};
+
+// 70 MB default max video upload size
+const MAX_FILE_SIZE_IN_MB = 70;
+const MAX_VIDEO_UPLOAD_SIZE = MAX_FILE_SIZE_IN_MB * 1024 ** 2;
+
+type Props = {
+  onUpload: (video: VideoData) => void;
+  onUploadStart?: () => void;
+  onUploadError?: (error: Error) => void;
+};
+
+export default function useUploadVideo({
+  onUpload,
+  onUploadStart,
+  onUploadError,
+}: Props) {
+  const [error, setError] = useState<string | null>(null);
+  const [commit, isMutationInFlight] = useMutation<useUploadVideoMutation>(
+    graphql`
+      mutation useUploadVideoMutation($file: Upload!) {
+        uploadVideo(file: $file) {
+          id
+          height
+          width
+          url
+          path
+          posterPath
+          posterUrl
+        }
+      }
+    `,
+  );
+
+  const {getRootProps, getInputProps} = useDropzone({
+    accept: ACCEPT_VIDEOS,
+    multiple: false,
+    maxFiles: 1,
+    onDrop: (
+      acceptedFiles: FileWithPath[],
+      fileRejections: FileRejection[],
+    ) => {
+      setError(null);
+
+      // Check if any of the files (only 1 file allowed) is rejected. The
+      // rejected file has an error (e.g., 'file-too-large'). Rendering an
+      // appropriate message.
+      if (fileRejections.length > 0 && fileRejections[0].errors.length > 0) {
+        const code = fileRejections[0].errors[0].code;
+        if (code === 'file-too-large') {
+          setError(
+            `File too large. Try a video under ${MAX_FILE_SIZE_IN_MB} MB`,
+          );
+          return;
+        }
+      }
+
+      if (acceptedFiles.length === 0) {
+        setError('File not accepted. Please try again.');
+        return;
+      }
+      if (acceptedFiles.length > 1) {
+        setError('Too many files. Please try again with 1 file.');
+        return;
+      }
+
+      onUploadStart?.();
+      const file = acceptedFiles[0];
+
+      commit({
+        variables: {
+          file,
+        },
+        uploadables: {
+          file,
+        },
+        onCompleted: response => onUpload(response.uploadVideo),
+        onError: error => {
+          Logger.error(error);
+          onUploadError?.(error);
+          setError('Upload failed.');
+        },
+      });
+    },
+    onError: error => {
+      Logger.error(error);
+      setError('File not supported.');
+    },
+    maxSize: MAX_VIDEO_UPLOAD_SIZE,
+  });
+
+  return {
+    getRootProps,
+    getInputProps,
+    isUploading: isMutationInFlight,
+    error,
+    setError,
+  };
+}

sam2/demo/frontend/src/common/components/icons/GitHubIcon.tsx

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type Props = {
+  className?: string;
+};
+
+export function GitHubIcon({className}: Props) {
+  return (
+    <svg viewBox="0 0 24 24" aria-hidden="true" className={className}>
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M12 2C6.477 2 2 6.463 2 11.97c0 4.404 2.865 8.14 6.839 9.458.5.092.682-.216.682-.48 0-.236-.008-.864-.013-1.695-2.782.602-3.369-1.337-3.369-1.337-.454-1.151-1.11-1.458-1.11-1.458-.908-.618.069-.606.069-.606 1.003.07 1.531 1.027 1.531 1.027.892 1.524 2.341 1.084 2.91.828.092-.643.35-1.083.636-1.332-2.22-.251-4.555-1.107-4.555-4.927 0-1.088.39-1.979 1.029-2.675-.103-.252-.446-1.266.098-2.638 0 0 .84-.268 2.75 1.022A9.607 9.607 0 0 1 12 6.82c.85.004 1.705.114 2.504.336 1.909-1.29 2.747-1.022 2.747-1.022.546 1.372.202 2.386.1 2.638.64.696 1.028 1.587 1.028 2.675 0 3.83-2.339 4.673-4.566 4.92.359.307.678.915.678 1.846 0 1.332-.012 2.407-.012 2.734 0 .267.18.577.688.48 3.97-1.32 6.833-5.054 6.833-9.458C22 6.463 17.522 2 12 2Z"></path>
+    </svg>
+  );
+}

sam2/demo/frontend/src/common/components/options/DownloadOption.tsx

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Package} from '@carbon/icons-react';
+import OptionButton from './OptionButton';
+import useDownloadVideo from './useDownloadVideo';
+
+export default function DownloadOption() {
+  const {download, state} = useDownloadVideo();
+
+  return (
+    <OptionButton
+      title="Download"
+      Icon={Package}
+      loadingProps={{
+        loading: state === 'started' || state === 'encoding',
+        label: 'Downloading...',
+      }}
+      onClick={download}
+    />
+  );
+}

sam2/demo/frontend/src/common/components/options/GalleryOption.tsx

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideoModal from '@/common/components/gallery/ChangeVideoModal';
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {ImageCopy} from '@carbon/icons-react';
+import OptionButton from './OptionButton';
+
+type Props = {
+  onChangeVideo: () => void;
+};
+export default function GalleryOption({onChangeVideo}: Props) {
+  return (
+    <ChangeVideoModal
+      videoGalleryModalTrigger={GalleryTrigger}
+      showUploadInGallery={false}
+      onChangeVideo={onChangeVideo}
+    />
+  );
+}
+
+function GalleryTrigger({onClick}: VideoGalleryTriggerProps) {
+  const {isMobile} = useScreenSize();
+
+  return (
+    <OptionButton
+      variant="flat"
+      title={isMobile ? 'Gallery' : 'Browse gallery'}
+      Icon={ImageCopy}
+      onClick={onClick}
+    />
+  );
+}

sam2/demo/frontend/src/common/components/options/MoreOptionsToolbar.tsx

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import MoreOptionsToolbarBottomActions from '@/common/components/options/MoreOptionsToolbarBottomActions';
+import ShareSection from '@/common/components/options/ShareSection';
+import TryAnotherVideoSection from '@/common/components/options/TryAnotherVideoSection';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {useEffect, useRef} from 'react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MoreOptionsToolbar({onTabChange}: Props) {
+  const {isMobile} = useScreenSize();
+  const {clearMessage} = useMessagesSnackbar();
+  const didClearMessageSnackbar = useRef(false);
+
+  useEffect(() => {
+    if (!didClearMessageSnackbar.current) {
+      didClearMessageSnackbar.current = true;
+      clearMessage();
+    }
+  }, [clearMessage]);
+
+  return (
+    <div className="flex flex-col h-full">
+      <div className="grow">
+        <ToolbarHeaderWrapper
+          title="Nice work! What's next?"
+          className="pb-0 !border-b-0 !text-white"
+          showProgressChip={false}
+        />
+        <ShareSection />
+        {!isMobile && <div className="h-[1px] bg-black mt-4 mb-8"></div>}
+        <TryAnotherVideoSection onTabChange={onTabChange} />
+      </div>
+      {!isMobile && (
+        <MoreOptionsToolbarBottomActions onTabChange={onTabChange} />
+      )}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/options/MoreOptionsToolbarBottomActions.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import RestartSessionButton from '@/common/components/session/RestartSessionButton';
+import {
+  EFFECT_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {ChevronLeft} from '@carbon/icons-react';
+import {Button} from 'react-daisyui';
+import ToolbarBottomActionsWrapper from '../toolbar/ToolbarBottomActionsWrapper';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MoreOptionsToolbarBottomActions({onTabChange}: Props) {
+  function handleReturnToEffectsTab() {
+    onTabChange(EFFECT_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <Button
+        color="ghost"
+        onClick={handleReturnToEffectsTab}
+        className="!px-4 !rounded-full font-medium text-white hover:bg-black"
+        startIcon={<ChevronLeft />}>
+        Edit effects
+      </Button>
+      <RestartSessionButton
+        onRestartSession={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+    </ToolbarBottomActionsWrapper>
+  );
+}

sam2/demo/frontend/src/common/components/options/OptionButton.tsx

@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {Loading} from 'react-daisyui';
+
+type Props = {
+  variant?: 'default' | 'flat' | 'gradient';
+  title: string | React.ReactNode;
+  Icon: CarbonIconType;
+  isActive?: boolean;
+  isDisabled?: boolean;
+  loadingProps?: {
+    loading: boolean;
+    label?: string;
+  };
+  onClick: () => void;
+};
+
+export default function OptionButton({
+  variant = 'default',
+  title,
+  Icon,
+  isActive = false,
+  isDisabled = false,
+  loadingProps,
+  onClick,
+}: Props) {
+  const {isMobile} = useScreenSize();
+  const isLoading = loadingProps?.loading === true;
+
+  function handleClick() {
+    if (isDisabled) {
+      return;
+    }
+    onClick();
+  }
+
+  const ButtonBase = (
+    <div
+      onClick={handleClick}
+      className={`relative rounded-lg h-full flex items-center justify-center 
+${variant === 'default' ? 'bg-graydark-700' : ''}
+${!isDisabled && 'cursor-pointer'}
+${isDisabled ? 'text-gray-300' : ''}
+${isActive && BLUE_PINK_FILL_BR}`}>
+      <div className="flex gap-2 items-center py-4 md:py-6">
+        {isLoading ? (
+          <Loading size="md" className="mx-auto mt-1" />
+        ) : (
+          <Icon
+            size={isMobile ? 24 : 28}
+            className={`mx-auto ${isDisabled ? 'text-gray-300' : 'text-white'}`}
+          />
+        )}
+        <div className="text-base font-medium text-white">
+          {isLoading && loadingProps?.label != null
+            ? loadingProps.label
+            : title}
+        </div>
+      </div>
+    </div>
+  );
+
+  return variant === 'gradient' ? (
+    <GradientBorder rounded={false} className={'rounded-lg md:rounded-full'}>
+      {ButtonBase}
+    </GradientBorder>
+  ) : (
+    ButtonBase
+  );
+}

sam2/demo/frontend/src/common/components/options/ShareSection.tsx

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import DownloadOption from './DownloadOption';
+
+export default function ShareSection() {
+  return (
+    <div className="p-5 md:p-8">
+      <DownloadOption />
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/options/ShareUtils.ts

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export async function handleSaveVideo(
+  videoPath: string,
+  fileName?: string,
+): Promise<void> {
+  const blob = await fetch(videoPath).then(res => res.blob());
+
+  return new Promise(resolve => {
+    const reader = new FileReader();
+    reader.readAsDataURL(blob);
+    reader.addEventListener('load', () => {
+      const elem = document.createElement('a');
+      elem.download = fileName ?? getFileName();
+      if (typeof reader.result === 'string') {
+        elem.href = reader.result;
+      }
+      elem.click();
+      resolve();
+    });
+  });
+}
+
+export function getFileName() {
+  const date = new Date();
+  const timestamp = date.getTime();
+  return `sam2_masked_video_${timestamp}.mp4`;
+}

sam2/demo/frontend/src/common/components/options/TryAnotherVideoSection.tsx

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GalleryOption from '@/common/components/options/GalleryOption';
+import UploadOption from '@/common/components/options/UploadOption';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useScreenSize from '@/common/screen/useScreenSize';
+
+type Props = {
+  onTabChange: (tabIndex: number) => void;
+};
+
+export default function TryAnotherVideoSection({onTabChange}: Props) {
+  const {isMobile} = useScreenSize();
+  const video = useVideo();
+
+  function handleVideoChange() {
+    if (video != null) {
+      video.pause();
+      video.frame = 0;
+    }
+    onTabChange(OBJECT_TOOLBAR_INDEX);
+  }
+
+  if (isMobile) {
+    return (
+      <div className="px-8 pb-8">
+        <div className="font-medium text-gray-300 text-sm">
+          Or, try another video
+        </div>
+        <div className="flex flex-row gap-4 mt-4 w-full">
+          <div className="flex-1">
+            <UploadOption onUpload={handleVideoChange} />
+          </div>
+          <div className="flex-1">
+            <GalleryOption onChangeVideo={handleVideoChange} />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
+  return (
+    <div className="px-8 pb-8">
+      <div className="font-medium text-gray-300 text-base">
+        Try another video
+      </div>
+      <div className="flex flex-col gap-4 mt-4">
+        <UploadOption onUpload={handleVideoChange} />
+        <GalleryOption onChangeVideo={handleVideoChange} />
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/options/UploadOption.tsx

@@ -0,0 +1,86 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useUploadVideo from '@/common/components/gallery/useUploadVideo';
+import OptionButton from '@/common/components/options/OptionButton';
+import Logger from '@/common/logger/Logger';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {sessionAtom, uploadingStateAtom} from '@/demo/atoms';
+import {MAX_UPLOAD_FILE_SIZE} from '@/demo/DemoConfig';
+import {Close, CloudUpload} from '@carbon/icons-react';
+import {useSetAtom} from 'jotai';
+import {useNavigate} from 'react-router-dom';
+
+type Props = {
+  onUpload: () => void;
+};
+
+export default function UploadOption({onUpload}: Props) {
+  const navigate = useNavigate();
+  const {isMobile} = useScreenSize();
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const setSession = useSetAtom(sessionAtom);
+
+  const {getRootProps, getInputProps, isUploading, error} = useUploadVideo({
+    onUpload: videoData => {
+      navigate(
+        {pathname: location.pathname, search: location.search},
+        {state: {video: videoData}},
+      );
+      onUpload();
+      setUploadingState('default');
+      setSession(null);
+    },
+    onUploadError: (error: Error) => {
+      setUploadingState('error');
+      Logger.error(error);
+    },
+    onUploadStart: () => {
+      setUploadingState('uploading');
+    },
+  });
+
+  return (
+    <div className="cursor-pointer" {...getRootProps()}>
+      <input {...getInputProps()} />
+
+      <OptionButton
+        variant="gradient"
+        title={
+          error !== null ? (
+            'Upload Error'
+          ) : isMobile ? (
+            <>
+              Upload{' '}
+              <div className="text-xs opacity-70">
+                Max {MAX_UPLOAD_FILE_SIZE}
+              </div>
+            </>
+          ) : (
+            <>
+              Upload your own{' '}
+              <div className="text-xs opacity-70">
+                Max {MAX_UPLOAD_FILE_SIZE}
+              </div>
+            </>
+          )
+        }
+        Icon={error !== null ? Close : CloudUpload}
+        loadingProps={{loading: isUploading, label: 'Uploading...'}}
+        onClick={() => {}}
+      />
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/options/__generated__/GetLinkOptionShareVideoMutation.graphql.ts

@@ -0,0 +1,89 @@
+/**
+ * @generated SignedSource<<39d7e92a6c15de1583c90ae21a7825e5>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type GetLinkOptionShareVideoMutation$variables = {
+  file: any;
+};
+export type GetLinkOptionShareVideoMutation$data = {
+  readonly uploadSharedVideo: {
+    readonly path: string;
+  };
+};
+export type GetLinkOptionShareVideoMutation = {
+  response: GetLinkOptionShareVideoMutation$data;
+  variables: GetLinkOptionShareVideoMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "file"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "file",
+        "variableName": "file"
+      }
+    ],
+    "concreteType": "SharedVideo",
+    "kind": "LinkedField",
+    "name": "uploadSharedVideo",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "path",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "GetLinkOptionShareVideoMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "GetLinkOptionShareVideoMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "f02ec81a41c8d75c3733853e1fb04f58",
+    "id": null,
+    "metadata": {},
+    "name": "GetLinkOptionShareVideoMutation",
+    "operationKind": "mutation",
+    "text": "mutation GetLinkOptionShareVideoMutation(\n  $file: Upload!\n) {\n  uploadSharedVideo(file: $file) {\n    path\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "c1b085da9afaac5f19eeb99ff561ed55";
+
+export default node;

sam2/demo/frontend/src/common/components/options/useDownloadVideo.ts

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {getFileName} from '@/common/components/options/ShareUtils';
+import {
+  EncodingCompletedEvent,
+  EncodingStateUpdateEvent,
+} from '@/common/components/video/VideoWorkerBridge';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {MP4ArrayBuffer} from 'mp4box';
+import {useState} from 'react';
+
+type DownloadingState = 'default' | 'started' | 'encoding' | 'completed';
+
+type State = {
+  state: DownloadingState;
+  progress: number;
+  download: (shouldSave?: boolean) => Promise<MP4ArrayBuffer>;
+};
+
+export default function useDownloadVideo(): State {
+  const [downloadingState, setDownloadingState] =
+    useState<DownloadingState>('default');
+  const [progress, setProgress] = useState<number>(0);
+
+  const video = useVideo();
+
+  async function download(shouldSave = true): Promise<MP4ArrayBuffer> {
+    return new Promise(resolve => {
+      function onEncodingStateUpdate(event: EncodingStateUpdateEvent) {
+        setDownloadingState('encoding');
+        setProgress(event.progress);
+      }
+
+      function onEncodingComplete(event: EncodingCompletedEvent) {
+        const file = event.file;
+
+        if (shouldSave) {
+          saveVideo(file, getFileName());
+        }
+
+        video?.removeEventListener('encodingCompleted', onEncodingComplete);
+        video?.removeEventListener(
+          'encodingStateUpdate',
+          onEncodingStateUpdate,
+        );
+        setDownloadingState('completed');
+        resolve(file);
+      }
+
+      video?.addEventListener('encodingStateUpdate', onEncodingStateUpdate);
+      video?.addEventListener('encodingCompleted', onEncodingComplete);
+
+      if (downloadingState === 'default' || downloadingState === 'completed') {
+        setDownloadingState('started');
+        video?.pause();
+        video?.encode();
+      }
+    });
+  }
+
+  function saveVideo(file: MP4ArrayBuffer, fileName: string) {
+    const blob = new Blob([file]);
+    const url = window.URL.createObjectURL(blob);
+
+    const a = document.createElement('a');
+    document.body.appendChild(a);
+    a.setAttribute('href', url);
+    a.setAttribute('download', fileName);
+    a.setAttribute('target', '_self');
+    a.click();
+    window.URL.revokeObjectURL(url);
+  }
+
+  return {download, progress, state: downloadingState};
+}

sam2/demo/frontend/src/common/components/session/RestartSessionButton.tsx

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useRestartSession from '@/common/components/session/useRestartSession';
+import {Reset} from '@carbon/icons-react';
+import {Button, Loading} from 'react-daisyui';
+
+type Props = {
+  onRestartSession: () => void;
+};
+
+export default function RestartSessionButton({onRestartSession}: Props) {
+  const {restartSession, isLoading} = useRestartSession();
+
+  function handleRestartSession() {
+    restartSession(onRestartSession);
+  }
+
+  return (
+    <Button
+      color="ghost"
+      onClick={handleRestartSession}
+      className="!px-4 !rounded-full font-medium text-white hover:bg-black"
+      startIcon={isLoading ? <Loading size="sm" /> : <Reset size={20} />}>
+      Start over
+    </Button>
+  );
+}

sam2/demo/frontend/src/common/components/session/__generated__/useCloseSessionBeforeUnloadMutation.graphql.ts

@@ -0,0 +1,92 @@
+/**
+ * @generated SignedSource<<f56872c0a8b65fa7e9bdaff351930ff0>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type CloseSessionInput = {
+  sessionId: string;
+};
+export type useCloseSessionBeforeUnloadMutation$variables = {
+  input: CloseSessionInput;
+};
+export type useCloseSessionBeforeUnloadMutation$data = {
+  readonly closeSession: {
+    readonly success: boolean;
+  };
+};
+export type useCloseSessionBeforeUnloadMutation = {
+  response: useCloseSessionBeforeUnloadMutation$data;
+  variables: useCloseSessionBeforeUnloadMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "CloseSession",
+    "kind": "LinkedField",
+    "name": "closeSession",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "success",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "useCloseSessionBeforeUnloadMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "useCloseSessionBeforeUnloadMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "99b73bd43a9f74104d545778cebbd15c",
+    "id": null,
+    "metadata": {},
+    "name": "useCloseSessionBeforeUnloadMutation",
+    "operationKind": "mutation",
+    "text": "mutation useCloseSessionBeforeUnloadMutation(\n  $input: CloseSessionInput!\n) {\n  closeSession(input: $input) {\n    success\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "55dd870645c9736b797b90819ddb1b92";
+
+export default node;

sam2/demo/frontend/src/common/components/session/useCloseSessionBeforeUnload.ts

@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useCloseSessionBeforeUnloadMutation$variables} from '@/common/components/session/__generated__/useCloseSessionBeforeUnloadMutation.graphql';
+import {sessionAtom} from '@/demo/atoms';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {useAtomValue} from 'jotai';
+import {useEffect, useMemo} from 'react';
+import {ConcreteRequest, graphql} from 'relay-runtime';
+
+/**
+ * The useCloseSessionBeforeUnload is a dirty workaround to send close session
+ * requests on window/tab close. Going through Relay does not send the request
+ * even if the `keepalive` flag is set for the request. It does work when the
+ * fetch is called directly with the close session mutation.
+ *
+ * Caveat: there is static typing, but there might be other caveats around this
+ * quirky hack.
+ */
+export default function useCloseSessionBeforeUnload() {
+  const session = useAtomValue(sessionAtom);
+  const {settings} = useSettingsContext();
+
+  const data = useMemo(() => {
+    if (session == null) {
+      return null;
+    }
+
+    const graphQLTaggedNode = graphql`
+      mutation useCloseSessionBeforeUnloadMutation($input: CloseSessionInput!) {
+        closeSession(input: $input) {
+          success
+        }
+      }
+    ` as ConcreteRequest;
+
+    const variables: useCloseSessionBeforeUnloadMutation$variables = {
+      input: {
+        sessionId: session.id,
+      },
+    };
+
+    const query = graphQLTaggedNode.params.text;
+    if (query === null) {
+      return null;
+    }
+
+    return {
+      query,
+      variables,
+    };
+  }, [session]);
+
+  useEffect(() => {
+    function onBeforeUpload() {
+      if (data == null) {
+        return;
+      }
+
+      fetch(`${settings.inferenceAPIEndpoint}/graphql`, {
+        method: 'POST',
+        credentials: 'include',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        keepalive: true,
+        body: JSON.stringify(data),
+      });
+    }
+    window.addEventListener('beforeunload', onBeforeUpload);
+    return () => {
+      window.removeEventListener('beforeunload', onBeforeUpload);
+    };
+  }, [data, session, settings.inferenceAPIEndpoint]);
+}

sam2/demo/frontend/src/common/components/session/useRestartSession.ts

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useInputVideo from '@/common/components/video/useInputVideo';
+import {
+  activeTrackletObjectIdAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+  labelTypeAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {useState} from 'react';
+
+export default function useRestartSession() {
+  const [isLoading, setIsLoading] = useState<boolean>();
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setActiveTrackletObjectId = useSetAtom(activeTrackletObjectIdAtom);
+  const setTracklets = useSetAtom(trackletObjectsAtom);
+  const setLabelType = useSetAtom(labelTypeAtom);
+  const {clearMessage} = useMessagesSnackbar();
+
+  const {inputVideo} = useInputVideo();
+  const video = useVideo();
+
+  async function restartSession(onRestart?: () => void) {
+    if (video === null || inputVideo === null) {
+      return;
+    }
+
+    setIsLoading(true);
+    if (isPlaying) {
+      video.pause();
+    }
+    if (isStreaming) {
+      await video.abortStreamMasks();
+    }
+    await video?.startSession(inputVideo.path);
+    video.frame = 0;
+    setActiveTrackletObjectId(0);
+    setTracklets([]);
+    setLabelType('positive');
+    onRestart?.();
+    clearMessage();
+    setIsLoading(false);
+  }
+
+  return {isLoading, restartSession};
+}

sam2/demo/frontend/src/common/components/snackbar/DemoMessagesSnackbarUtils.ts

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {EnqueueOption} from '@/common/components/snackbar/useMessagesSnackbar';
+
+export type MessageOptions = EnqueueOption & {
+  repeat?: boolean;
+};
+
+type MessageEvent = {
+  text: string;
+  shown: boolean;
+  action?: Element;
+  options?: MessageOptions;
+};
+
+export interface MessagesEventMap {
+  startSession: MessageEvent;
+  firstClick: MessageEvent;
+  pointClick: MessageEvent;
+  addObjectClick: MessageEvent;
+  trackAndPlayClick: MessageEvent;
+  trackAndPlayComplete: MessageEvent;
+  trackAndPlayThrottlingWarning: MessageEvent;
+  effectsMessage: MessageEvent;
+}
+
+export const defaultMessageMap: MessagesEventMap = {
+  startSession: {
+    text: 'Starting session',
+    shown: false,
+    options: {type: 'loading', showClose: false, repeat: true, duration: 2000},
+  },
+  firstClick: {
+    text: 'Tip: Click on any object in the video to get started.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  pointClick: {
+    text: 'Tip: Not what you expected? Add a few more clicks until the full object you want is selected.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  addObjectClick: {
+    text: 'Tip: Add a new object by clicking on it in the video.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  trackAndPlayClick: {
+    text: 'Hang tight while your objects are tracked! You’ll be able to apply visual effects in the next step. Stop tracking at any point to adjust your selections if the tracking doesn’t look right.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  trackAndPlayComplete: {
+    text: 'Tip: You can fix tracking issues by going back to the frames where tracking is not quite right and adding or removing clicks.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+  trackAndPlayThrottlingWarning: {
+    text: 'Looks like you have clicked the tracking button a bit too often! To keep things running smoothly, we have temporarily disabled the button.',
+    shown: false,
+    options: {repeat: true},
+  },
+  effectsMessage: {
+    text: 'Tip: If you aren’t sure where to get started, click “Surprise Me” to apply a surprise effect to your video.',
+    shown: false,
+    options: {expire: false, repeat: false},
+  },
+};

sam2/demo/frontend/src/common/components/snackbar/MessagesSnackbar.tsx

@@ -0,0 +1,118 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useScreenSize from '@/common/screen/useScreenSize';
+import {color, gradients} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue} from 'jotai';
+import {Loading, RadialProgress} from 'react-daisyui';
+import {messageAtom} from './snackbarAtoms';
+import useExpireMessage from './useExpireMessage';
+import useMessagesSnackbar from './useMessagesSnackbar';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    top: '8px',
+    right: '8px',
+  },
+  mobileContainer: {
+    position: 'absolute',
+    bottom: '8px',
+    left: '8px',
+    right: '8px',
+  },
+  messageContainer: {
+    padding: '20px 20px',
+    color: '#FFF',
+    borderRadius: '8px',
+    fontSize: '0.9rem',
+    maxWidth: 400,
+    border: '2px solid transparent',
+    background: gradients['yellowTeal'],
+  },
+  messageWarningContainer: {
+    background: '#FFDC32',
+    color: color['gray-900'],
+  },
+  messageContent: {
+    display: 'flex',
+    alignItems: 'center',
+    gap: '8px',
+  },
+  progress: {
+    flexShrink: 0,
+    color: 'rgba(255, 255, 255, 0.1)',
+  },
+  closeColumn: {
+    display: 'flex',
+    alignSelf: 'stretch',
+    alignItems: 'start',
+  },
+});
+
+export default function MessagesSnackbar() {
+  const message = useAtomValue(messageAtom);
+  const {clearMessage} = useMessagesSnackbar();
+  const {isMobile} = useScreenSize();
+
+  useExpireMessage();
+
+  if (message == null) {
+    return null;
+  }
+
+  const closeIcon = (
+    <Close
+      size={24}
+      color={message.type === 'warning' ? color['gray-900'] : 'white'}
+      opacity={1}
+      className="z-20 hover:text-gray-300 color-white cursor-pointer !opacity-100 shrink-0"
+      onClick={clearMessage}
+    />
+  );
+
+  return (
+    <div
+      {...stylex.props(isMobile ? styles.mobileContainer : styles.container)}>
+      <div
+        {...stylex.props(
+          styles.messageContainer,
+          message.type === 'warning' && styles.messageWarningContainer,
+        )}>
+        <div {...stylex.props(styles.messageContent)}>
+          <div>{message.text}</div>
+          {message.type === 'loading' && <Loading size="xs" variant="dots" />}
+          {message.showClose && (
+            <div {...stylex.props(styles.closeColumn)}>
+              {message.expire ? (
+                <RadialProgress
+                  value={message.progress * 100}
+                  size="32px"
+                  thickness="2px"
+                  {...stylex.props(styles.progress)}>
+                  {closeIcon}
+                </RadialProgress>
+              ) : (
+                closeIcon
+              )}
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/snackbar/snackbarAtoms.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+
+export type MessageType = 'info' | 'loading' | 'warning';
+
+export type Message = {
+  type: MessageType;
+  text: string;
+  duration: number;
+  progress: number;
+  startTime: number;
+  expire: boolean;
+  showClose: boolean;
+  showReset: boolean;
+};
+
+export const messageAtom = atom<Message | null>(null);

sam2/demo/frontend/src/common/components/snackbar/useDemoMessagesSnackbar.ts

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {MessagesEventMap} from '@/common/components/snackbar/DemoMessagesSnackbarUtils';
+import useMessagesSnackbar from '@/common/components/snackbar/useMessagesSnackbar';
+import {messageMapAtom} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+import {useCallback} from 'react';
+
+type State = {
+  enqueueMessage: (messageType: keyof MessagesEventMap) => void;
+  clearMessage: () => void;
+};
+
+export default function useDemoMessagesSnackbar(): State {
+  const [messageMap, setMessageMap] = useAtom(messageMapAtom);
+  const {enqueueMessage: enqueue, clearMessage} = useMessagesSnackbar();
+
+  const enqueueMessage = useCallback(
+    (messageType: keyof MessagesEventMap) => {
+      const {text, shown, options} = messageMap[messageType];
+      if (!options?.repeat && shown === true) {
+        return;
+      }
+      enqueue(text, options);
+      const newState = {...messageMap};
+      newState[messageType].shown = true;
+      setMessageMap(newState);
+    },
+    [enqueue, messageMap, setMessageMap],
+  );
+
+  return {enqueueMessage, clearMessage};
+}

sam2/demo/frontend/src/common/components/snackbar/useExpireMessage.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useAtom} from 'jotai';
+import {useEffect, useRef} from 'react';
+import {Message, messageAtom} from '@/common/components/snackbar/snackbarAtoms';
+
+export default function useExpireMessage() {
+  const [message, setMessage] = useAtom(messageAtom);
+  const messageRef = useRef<Message | null>(null);
+  const intervalRef = useRef<NodeJS.Timeout | null>(null);
+
+  useEffect(() => {
+    messageRef.current = message;
+  }, [message]);
+
+  useEffect(() => {
+    function resetInterval() {
+      if (intervalRef.current != null) {
+        clearInterval(intervalRef.current);
+        intervalRef.current = null;
+      }
+    }
+    if (intervalRef.current == null && message != null && message.expire) {
+      intervalRef.current = setInterval(() => {
+        const prevMessage = messageRef.current;
+        if (prevMessage == null) {
+          setMessage(null);
+          resetInterval();
+          return;
+        }
+        const messageDuration = Date.now() - prevMessage.startTime;
+        if (messageDuration > prevMessage.duration) {
+          setMessage(null);
+          resetInterval();
+          return;
+        }
+        setMessage({
+          ...prevMessage,
+          progress: messageDuration / prevMessage.duration,
+        });
+      }, 20);
+    }
+  }, [message, setMessage]);
+
+  useEffect(() => {
+    return () => {
+      if (intervalRef.current != null) {
+        clearInterval(intervalRef.current);
+      }
+    };
+  }, []);
+}

sam2/demo/frontend/src/common/components/snackbar/useMessagesSnackbar.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useSetAtom} from 'jotai';
+import {useCallback} from 'react';
+import {
+  MessageType,
+  messageAtom,
+} from '@/common/components/snackbar/snackbarAtoms';
+
+export type EnqueueOption = {
+  duration?: number;
+  type?: MessageType;
+  expire?: boolean;
+  showClose?: boolean;
+  showReset?: boolean;
+};
+
+type State = {
+  clearMessage: () => void;
+  enqueueMessage: (message: string, options?: EnqueueOption) => void;
+};
+
+export default function useMessagesSnackbar(): State {
+  const setMessage = useSetAtom(messageAtom);
+
+  const enqueueMessage = useCallback(
+    (message: string, options?: EnqueueOption) => {
+      setMessage({
+        text: message,
+        type: options?.type ?? 'info',
+        duration: options?.duration ?? 5000,
+        progress: 0,
+        startTime: Date.now(),
+        expire: options?.expire ?? true,
+        showClose: options?.showClose ?? true,
+        showReset: options?.showReset ?? false,
+      });
+    },
+    [setMessage],
+  );
+
+  function clearMessage() {
+    setMessage(null);
+  }
+
+  return {enqueueMessage, clearMessage};
+}

sam2/demo/frontend/src/common/components/toolbar/DesktopToolbar.tsx

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ObjectsToolbar from '@/common/components/annotations/ObjectsToolbar';
+import EffectsToolbar from '@/common/components/effects/EffectsToolbar';
+import MoreOptionsToolbar from '@/common/components/options/MoreOptionsToolbar';
+import type {CSSProperties} from 'react';
+
+type Props = {
+  tabIndex: number;
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function DesktopToolbar({tabIndex, onTabChange}: Props) {
+  const toolbarShadow: CSSProperties = {
+    boxShadow: '0px 1px 3px 1px rgba(0,0,0,.25)',
+    transition: 'box-shadow 0.8s ease-out',
+  };
+
+  const tabs = [
+    <ObjectsToolbar key="objects" onTabChange={onTabChange} />,
+    <EffectsToolbar key="effects" onTabChange={onTabChange} />,
+    <MoreOptionsToolbar key="options" onTabChange={onTabChange} />,
+  ];
+
+  return (
+    <div
+      style={toolbarShadow}
+      className="bg-graydark-800 text-white md:basis-[350px] lg:basis-[435px] shrink-0 rounded-xl">
+      {tabs[tabIndex]}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/toolbar/MobileToolbar.tsx

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import MobileObjectsToolbar from '@/common/components/annotations/MobileObjectsToolbar';
+import MobileEffectsToolbar from '@/common/components/effects/MobileEffectsToolbar';
+import MoreOptionsToolbar from '@/common/components/options/MoreOptionsToolbar';
+
+type Props = {
+  tabIndex: number;
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function MobileToolbar({tabIndex, onTabChange}: Props) {
+  const tabs = [
+    <MobileObjectsToolbar key="objects" onTabChange={onTabChange} />,
+    <MobileEffectsToolbar key="effects" onTabChange={onTabChange} />,
+    <MoreOptionsToolbar key="more-options" onTabChange={onTabChange} />,
+  ];
+
+  return (
+    <div className="relative flex flex-col bg-black">{tabs[tabIndex]}</div>
+  );
+}

sam2/demo/frontend/src/common/components/toolbar/Toolbar.tsx

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useListenToStreamingState from '@/common/components/toolbar/useListenToStreamingState';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {
+  codeEditorOpenedAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+} from '@/demo/atoms';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+import DesktopToolbar from './DesktopToolbar';
+import MobileToolbar from './MobileToolbar';
+import {OBJECT_TOOLBAR_INDEX} from './ToolbarConfig';
+
+export default function Toolbar() {
+  const [tabIndex, setTabIndex] = useToolbarTabs();
+  const video = useVideo();
+  const setIsPlaying = useSetAtom(isPlayingAtom);
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const codeEditorOpened = useAtomValue(codeEditorOpenedAtom);
+  const {isMobile} = useScreenSize();
+  const setEffect = useVideoEffect();
+
+  const resetEffects = useCallback(() => {
+    setEffect('Original', EffectIndex.BACKGROUND, {variant: 0});
+    setEffect('Overlay', EffectIndex.HIGHLIGHT, {variant: 0});
+  }, [setEffect]);
+
+  const handleStopVideo = useCallback(() => {
+    if (isStreaming) {
+      video?.abortStreamMasks();
+    } else {
+      video?.pause();
+    }
+  }, [video, isStreaming]);
+
+  const handleTabChange = useCallback(
+    (newIndex: number) => {
+      if (newIndex === OBJECT_TOOLBAR_INDEX) {
+        handleStopVideo();
+        resetEffects();
+      }
+
+      setTabIndex(newIndex);
+    },
+    [handleStopVideo, resetEffects, setTabIndex],
+  );
+
+  useListenToStreamingState();
+
+  useEffect(() => {
+    function onPlay() {
+      setIsPlaying(true);
+    }
+    function onPause() {
+      setIsPlaying(false);
+    }
+
+    video?.addEventListener('play', onPlay);
+    video?.addEventListener('pause', onPause);
+    return () => {
+      video?.removeEventListener('play', onPlay);
+      video?.removeEventListener('pause', onPause);
+    };
+  }, [video, resetEffects, setIsStreaming, setIsPlaying]);
+
+  if (codeEditorOpened) {
+    return null;
+  }
+
+  return isMobile ? (
+    <MobileToolbar tabIndex={tabIndex} onTabChange={handleTabChange} />
+  ) : (
+    <DesktopToolbar tabIndex={tabIndex} onTabChange={handleTabChange} />
+  );
+}

sam2/demo/frontend/src/common/components/toolbar/ToolbarActionIcon.tsx

@@ -0,0 +1,99 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {Loading} from 'react-daisyui';
+
+type Props = {
+  isDisabled?: boolean;
+  isActive?: boolean;
+  icon: CarbonIconType;
+  title: string;
+  badge?: React.ReactNode;
+  variant: 'toggle' | 'button' | 'gradient' | 'flat';
+  span?: 1 | 2;
+  loadingProps?: {
+    loading: boolean;
+    label?: string;
+  };
+  onClick: () => void;
+};
+
+export default function ToolbarActionIcon({
+  variant,
+  isDisabled = false,
+  isActive = false,
+  title,
+  badge,
+  loadingProps,
+  icon: Icon,
+  span = 1,
+  onClick,
+}: Props) {
+  const {isMobile} = useScreenSize();
+  const isLoading = loadingProps?.loading === true;
+
+  function handleClick() {
+    if (isDisabled) {
+      return;
+    }
+    onClick();
+  }
+
+  const ButtonBase = (
+    <div
+      onClick={handleClick}
+      className={`relative rounded-lg h-full flex items-center justify-center select-none
+      ${!isDisabled && 'cursor-pointer hover:bg-black'}
+      ${span === 1 && 'col-span-1'} 
+      ${span === 2 && 'col-span-2'}
+      ${variant === 'button' && (isDisabled ? 'bg-graydark-500 text-gray-300' : 'bg-graydark-700 hover:bg-graydark-800 text-white')} 
+      ${variant === 'toggle' && (isActive ? BLUE_PINK_FILL_BR : 'bg-inherit')}
+      ${variant === 'flat' && (isDisabled ? ' text-gray-600' : 'text-white')} 
+      `}>
+      <div className="py-4 px-2">
+        <div className="flex items-center justify-center">
+          {isLoading ? (
+            <Loading size="md" className="mx-auto" />
+          ) : (
+            <Icon
+              size={isMobile ? 24 : 28}
+              color={isActive ? 'white' : 'black'}
+              className={`mx-auto ${isDisabled ? 'text-gray-300' : 'text-white'}`}
+            />
+          )}
+        </div>
+        <div
+          className={`mt-1 md:mt-2 text-center text-xs font-bold ${isActive && 'text-white'}`}>
+          {isLoading && loadingProps?.label != null
+            ? loadingProps.label
+            : title}
+        </div>
+        {isActive && badge}
+      </div>
+    </div>
+  );
+
+  return variant == 'gradient' ? (
+    <GradientBorder rounded={false} className="rounded-lg h-full text-white">
+      {ButtonBase}
+    </GradientBorder>
+  ) : (
+    ButtonBase
+  );
+}

sam2/demo/frontend/src/common/components/toolbar/ToolbarBottomActionsWrapper.tsx

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren} from 'react';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+    justifyContent: 'space-between',
+    paddingTop: {
+      default: spacing[2],
+      '@media screen and (max-width: 768px)': spacing[4],
+    },
+    paddingBottom: spacing[6],
+    paddingHorizontal: spacing[6],
+  },
+});
+
+export default function ToolbarBottomActionsWrapper({
+  children,
+}: PropsWithChildren) {
+  return <div {...stylex.props(styles.container)}>{children}</div>;
+}

sam2/demo/frontend/src/common/components/toolbar/ToolbarConfig.tsx

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export const OBJECT_TOOLBAR_INDEX = 0;
+export const EFFECT_TOOLBAR_INDEX = 1;
+export const MORE_OPTIONS_TOOLBAR_INDEX = 2;

sam2/demo/frontend/src/common/components/toolbar/ToolbarHeaderWrapper.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {ReactNode} from 'react';
+import ToolbarProgressChip from './ToolbarProgressChip';
+
+type Props = {
+  title: string;
+  description?: string;
+  bottomSection?: ReactNode;
+  showProgressChip?: boolean;
+  className?: string;
+};
+
+export default function ToolbarHeaderWrapper({
+  title,
+  description,
+  bottomSection,
+  showProgressChip = true,
+  className,
+}: Props) {
+  return (
+    <div
+      className={`flex flex-col gap-2 p-8 border-b border-b-black ${className}`}>
+      <div className="flex items-center">
+        {showProgressChip && <ToolbarProgressChip />}
+        <h2 className="text-xl">{title}</h2>
+      </div>
+
+      {description != null && (
+        <div className="flex-1 text-gray-400">{description}</div>
+      )}
+      {bottomSection != null && bottomSection}
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/toolbar/ToolbarProgressChip.tsx

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import {streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+import {useMemo} from 'react';
+import {Loading} from 'react-daisyui';
+
+const TOTAL_DEMO_STEPS = 3;
+
+export default function ToolbarProgressChip() {
+  const [toolbarIndex] = useToolbarTabs();
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  const showLoader = useMemo(() => {
+    return streamingState === 'partial' || streamingState === 'requesting';
+  }, [streamingState]);
+
+  function getStepValue() {
+    if (toolbarIndex === OBJECT_TOOLBAR_INDEX) {
+      return streamingState !== 'full' ? 1 : 2;
+    }
+    return 3;
+  }
+
+  return (
+    <span className="inline-flex items-center justify-center rounded-full text-xs md:text-sm font-medium bg-white text-black w-10 md:w-12 h-5 md:h-6 mr-2 shrink-0 ">
+      {showLoader ? (
+        <Loading className="w-2 md:w-4" />
+      ) : (
+        `${getStepValue()}/${TOTAL_DEMO_STEPS}`
+      )}
+    </span>
+  );
+}

sam2/demo/frontend/src/common/components/toolbar/ToolbarSection.tsx

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {PropsWithChildren} from 'react';
+
+type Props = PropsWithChildren<{
+  title: string;
+  borderBottom?: boolean;
+}>;
+
+export default function ToolbarSection({
+  children,
+  title,
+  borderBottom = false,
+}: Props) {
+  return (
+    <div className={`p-6 ${borderBottom && 'border-b border-black'}`}>
+      <div className="font-bold ml-2">{title}</div>
+      <div className="grid grid-cols-4 gap-2 mt-2 md:mt-6">{children}</div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/toolbar/useListenToStreamingState.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {StreamingStateUpdateEvent} from '@/common/components/video/VideoWorkerBridge';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {StreamingState} from '@/common/tracker/Tracker';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+import {useEffect} from 'react';
+
+export default function useListenToStreamingState(): {
+  isStreaming: boolean;
+  streamingState: StreamingState;
+} {
+  const [streamingState, setStreamingState] = useAtom(streamingStateAtom);
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const video = useVideo();
+
+  useEffect(() => {
+    function onStreamingStateUpdate(event: StreamingStateUpdateEvent) {
+      setStreamingState(event.state);
+    }
+    function onStreamingStarted() {
+      setIsStreaming(true);
+    }
+    function onStreamingCompleted() {
+      setIsStreaming(false);
+    }
+    video?.addEventListener('streamingStateUpdate', onStreamingStateUpdate);
+    video?.addEventListener('streamingStarted', onStreamingStarted);
+    video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+    return () => {
+      video?.removeEventListener(
+        'streamingStateUpdate',
+        onStreamingStateUpdate,
+      );
+      video?.removeEventListener('streamingStarted', onStreamingStarted);
+      video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+    };
+  }, [video, setStreamingState, setIsStreaming]);
+
+  return {isStreaming, streamingState};
+}

sam2/demo/frontend/src/common/components/toolbar/useToolbarTabs.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {toolbarTabIndex} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+
+type State = [tabIndex: number, setTabIndex: (tabIndex: number) => void];
+
+export default function useToolbarTabs(): State {
+  return useAtom(toolbarTabIndex);
+}

sam2/demo/frontend/src/common/components/useFunctionThrottle.tsx

@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useCallback, useState} from 'react';
+
+type ThrottleOptions = {
+  enableThrottling?: boolean;
+};
+type State = {
+  isThrottled: boolean;
+  maxThrottles: boolean;
+  throttle: (callback: () => void, options?: ThrottleOptions) => void;
+};
+
+export default function useFunctionThrottle(
+  initialDelay: number,
+  numThrottles: number,
+): State {
+  const [isThrottled, setIsThrottled] = useState<boolean>(false);
+  const [lastClickTime, setLastClickTime] = useState<number | null>(null);
+  const [numTimesThrottled, setNumTimesThrottled] = useState<number>(1);
+
+  /**
+   * The following function's callback gets throttled when the time between two
+   * executions is less than a threshold.
+   *
+   * The threshold is calculated linearly by multiplying the initial delay
+   * and the number of times the button has been throttled. The button can be
+   * throttled up to numThrottles times.
+   *
+   * The function has an optional flag - enableThrottling - which allows a callsite
+   * to optionally disable throttling. This is useful in cases where throttling may
+   * not be necessary. (e.g. for the Track & Play button, we would only like to
+   * throttle after a stream is aborted.)
+   */
+  const throttle = useCallback(
+    (
+      callback: () => void,
+      options: ThrottleOptions = {
+        enableThrottling: true,
+      },
+    ) => {
+      if (isThrottled) {
+        return;
+      }
+
+      const currentTime = Date.now();
+      if (lastClickTime == null) {
+        callback();
+        setLastClickTime(currentTime);
+        return;
+      }
+
+      const timeBetweenClicks = currentTime - lastClickTime;
+      const delay = initialDelay * numTimesThrottled;
+      const shouldThrottle =
+        options.enableThrottling && delay > timeBetweenClicks;
+
+      if (shouldThrottle) {
+        setIsThrottled(true);
+        setTimeout(() => {
+          setIsThrottled(false);
+        }, delay);
+        setNumTimesThrottled(prev => {
+          return prev === numThrottles ? numThrottles : prev + 1;
+        });
+      }
+
+      callback();
+      setLastClickTime(currentTime);
+    },
+    [initialDelay, numThrottles, isThrottled, lastClickTime, numTimesThrottled],
+  );
+
+  return {
+    isThrottled,
+    maxThrottles: numTimesThrottled === numThrottles,
+    throttle,
+  };
+}

sam2/demo/frontend/src/common/components/video/ChangeVideoModal.tsx

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import DemoVideoGalleryModal from '@/common/components/gallery/DemoVideoGalleryModal';
+import useVideo from '@/common/components/video/editor/useVideo';
+import Logger from '@/common/logger/Logger';
+import {isStreamingAtom, uploadingStateAtom, VideoData} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {ComponentType, useCallback} from 'react';
+import {useNavigate} from 'react-router-dom';
+
+type Props = {
+  videoGalleryModalTrigger?: ComponentType<VideoGalleryTriggerProps>;
+  showUploadInGallery?: boolean;
+  onChangeVideo?: () => void;
+};
+
+export default function ChangeVideoModal({
+  videoGalleryModalTrigger: VideoGalleryModalTriggerComponent,
+  showUploadInGallery = true,
+  onChangeVideo,
+}: Props) {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const video = useVideo();
+  const navigate = useNavigate();
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  function handlePauseOrAbortVideo() {
+    if (isStreaming) {
+      video?.abortStreamMasks();
+    } else {
+      handlePause();
+    }
+  }
+
+  function handleSwitchVideos(video: VideoData) {
+    // Retain any search parameter
+    navigate(
+      {
+        pathname: location.pathname,
+        search: location.search,
+      },
+      {
+        state: {
+          video,
+        },
+      },
+    );
+    onChangeVideo?.();
+  }
+
+  function handleUploadVideoError(error: Error) {
+    setUploadingState('error');
+    Logger.error(error);
+  }
+
+  return (
+    <DemoVideoGalleryModal
+      trigger={VideoGalleryModalTriggerComponent}
+      showUploadInGallery={showUploadInGallery}
+      onOpen={handlePauseOrAbortVideo}
+      onSelect={handleSwitchVideos}
+      onUploadVideoError={handleUploadVideoError}
+    />
+  );
+}

sam2/demo/frontend/src/common/components/video/EventEmitter.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type EventMap<WorkerEventMap> = {
+  type: keyof WorkerEventMap;
+  listener: (ev: WorkerEventMap[keyof WorkerEventMap]) => unknown;
+};
+
+export class EventEmitter<WorkerEventMap> {
+  listeners: EventMap<WorkerEventMap>[] = [];
+
+  trigger<K extends keyof WorkerEventMap>(type: K, ev: WorkerEventMap[K]) {
+    this.listeners
+      .filter(listener => type === listener.type)
+      .forEach(({listener}) => {
+        setTimeout(() => listener(ev), 0);
+      });
+  }
+
+  addEventListener<K extends keyof WorkerEventMap>(
+    type: K,
+    listener: (ev: WorkerEventMap[K]) => unknown,
+  ): void {
+    // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+    this.listeners.push({type, listener});
+  }
+
+  removeEventListener<K extends keyof WorkerEventMap>(
+    type: K,
+    listener: (ev: WorkerEventMap[K]) => unknown,
+  ): void {
+    this.listeners = this.listeners.filter(
+      existingListener =>
+        !(
+          existingListener.type === type &&
+          existingListener.listener === listener
+        ),
+    );
+  }
+
+  destroy() {
+    this.listeners.length = 0;
+  }
+}

sam2/demo/frontend/src/common/components/video/Video.tsx

@@ -0,0 +1,374 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BaseTracklet, SegmentationPoint} from '@/common/tracker/Tracker';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import stylex, {StyleXStyles} from '@stylexjs/stylex';
+import {
+  CSSProperties,
+  forwardRef,
+  useEffect,
+  useImperativeHandle,
+  useMemo,
+  useRef,
+} from 'react';
+import {Button} from 'react-daisyui';
+
+import {EffectIndex, Effects} from '@/common/components/video/effects/Effects';
+import useReportError from '@/common/error/useReportError';
+import Logger from '@/common/logger/Logger';
+import {isPlayingAtom, isVideoLoadingAtom} from '@/demo/atoms';
+import {color} from '@/theme/tokens.stylex';
+import {useAtom} from 'jotai';
+import useResizeObserver from 'use-resize-observer';
+import VideoLoadingOverlay from './VideoLoadingOverlay';
+import {
+  StreamingStateUpdateEvent,
+  VideoWorkerEventMap,
+} from './VideoWorkerBridge';
+import {EffectOptions} from './effects/Effect';
+import useVideoWorker from './useVideoWorker';
+
+const styles = stylex.create({
+  container: {
+    position: 'relative',
+    width: '100%',
+    height: '100%',
+  },
+  canvasContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    backgroundColor: color['gray-800'],
+    width: '100%',
+    height: '100%',
+  },
+  controls: {
+    position: 'absolute',
+    bottom: 0,
+    left: 0,
+    width: '100%',
+    padding: 8,
+    background: 'linear-gradient(#00000000, #000000ff)',
+  },
+  controlButton: {
+    color: 'white',
+  },
+});
+
+type Props = {
+  src: string;
+  width: number;
+  height: number;
+  loading?: boolean;
+  containerStyle?: StyleXStyles<{
+    position: CSSProperties['position'];
+  }>;
+  canvasStyle?: StyleXStyles<{
+    width: CSSProperties['width'];
+  }>;
+  controls?: boolean;
+  createVideoWorker?: () => Worker;
+};
+
+export type VideoRef = {
+  getCanvas(): HTMLCanvasElement | null;
+  get width(): number;
+  get height(): number;
+  get frame(): number;
+  set frame(index: number);
+  get numberOfFrames(): number;
+  play(): void;
+  pause(): void;
+  stop(): void;
+  previousFrame(): void;
+  nextFrame(): void;
+  setEffect(
+    name: keyof Effects,
+    index: EffectIndex,
+    options?: EffectOptions,
+  ): void;
+  encode(): void;
+  streamMasks(): void;
+  abortStreamMasks(): Promise<void>;
+  addEventListener<K extends keyof VideoWorkerEventMap>(
+    type: K,
+    listener: (ev: VideoWorkerEventMap[K]) => unknown,
+  ): void;
+  removeEventListener<K extends keyof VideoWorkerEventMap>(
+    type: K,
+    listener: (ev: VideoWorkerEventMap[K]) => unknown,
+  ): void;
+  createFilmstrip(width: number, height: number): Promise<ImageBitmap>;
+  // Tracker
+  initializeTracker(name: keyof Trackers, options?: TrackerOptions): void;
+  startSession(videoUrl: string): Promise<string | null>;
+  closeSession(): void;
+  logAnnotations(): void;
+  createTracklet(): Promise<BaseTracklet>;
+  deleteTracklet(trackletId: number): Promise<void>;
+  updatePoints(trackletId: number, points: SegmentationPoint[]): void;
+  clearPointsInVideo(): Promise<boolean>;
+  getWorker_ONLY_USE_WITH_CAUTION(): Worker;
+};
+
+export default forwardRef<VideoRef, Props>(function Video(
+  {
+    src,
+    width,
+    height,
+    containerStyle,
+    canvasStyle,
+    createVideoWorker,
+    controls = false,
+    loading = false,
+  },
+  ref,
+) {
+  const reportError = useReportError();
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const [isPlaying, setIsPlaying] = useAtom(isPlayingAtom);
+  const [isVideoLoading, setIsVideoLoading] = useAtom(isVideoLoadingAtom);
+
+  const bridge = useVideoWorker(src, canvasRef, {
+    createVideoWorker,
+  });
+
+  const {
+    ref: resizeObserverRef,
+    width: resizeWidth = 1,
+    height: resizeHeight = 1,
+  } = useResizeObserver<HTMLDivElement>();
+
+  const canvasHeight = useMemo(() => {
+    const resizeRatio = resizeWidth / width;
+    return Math.min(height * resizeRatio, resizeHeight);
+  }, [resizeWidth, height, width, resizeHeight]);
+
+  useImperativeHandle(
+    ref,
+    () => ({
+      getCanvas() {
+        return canvasRef.current;
+      },
+      get width() {
+        return bridge.width;
+      },
+      get height() {
+        return bridge.width;
+      },
+      get frame() {
+        return bridge.frame;
+      },
+      set frame(index: number) {
+        bridge.frame = index;
+      },
+      get numberOfFrames() {
+        return bridge.numberOfFrames;
+      },
+      play(): void {
+        bridge.play();
+      },
+      pause(): void {
+        bridge.pause();
+      },
+      stop(): void {
+        bridge.stop();
+      },
+      previousFrame(): void {
+        bridge.previousFrame();
+      },
+      nextFrame(): void {
+        bridge.nextFrame();
+      },
+      setEffect(
+        name: keyof Effects,
+        index: number,
+        options?: EffectOptions,
+      ): void {
+        bridge.setEffect(name, index, options);
+      },
+      encode(): void {
+        bridge.encode();
+      },
+      streamMasks(): void {
+        bridge.streamMasks();
+      },
+      abortStreamMasks(): Promise<void> {
+        return bridge.abortStreamMasks();
+      },
+      addEventListener<K extends keyof VideoWorkerEventMap>(
+        type: K,
+        listener: (ev: VideoWorkerEventMap[K]) => unknown,
+      ): void {
+        bridge.addEventListener(type, listener);
+      },
+      removeEventListener<K extends keyof VideoWorkerEventMap>(
+        type: K,
+        listener: (ev: VideoWorkerEventMap[K]) => unknown,
+      ): void {
+        bridge.removeEventListener(type, listener);
+      },
+      createFilmstrip(width: number, height: number): Promise<ImageBitmap> {
+        return bridge.createFilmstrip(width, height);
+      },
+      // Tracker
+      initializeTracker(name: keyof Trackers, options: TrackerOptions): void {
+        bridge.initializeTracker(name, options);
+      },
+      startSession(videoUrl: string): Promise<string | null> {
+        return bridge.startSession(videoUrl);
+      },
+      closeSession(): void {
+        bridge.closeSession();
+      },
+      logAnnotations(): void {
+        bridge.logAnnotations();
+      },
+      createTracklet(): Promise<BaseTracklet> {
+        return bridge.createTracklet();
+      },
+      deleteTracklet(trackletId: number): Promise<void> {
+        return bridge.deleteTracklet(trackletId);
+      },
+      updatePoints(trackletId: number, points: SegmentationPoint[]): void {
+        bridge.updatePoints(trackletId, points);
+      },
+      clearPointsInVideo(): Promise<boolean> {
+        return bridge.clearPointsInVideo();
+      },
+      getWorker_ONLY_USE_WITH_CAUTION() {
+        return bridge.getWorker_ONLY_USE_WITH_CAUTION();
+      },
+    }),
+    [bridge],
+  );
+
+  // Handle video playback events (get playback state to main thread)
+  useEffect(() => {
+    let isPlaying = false;
+
+    function onFocus() {
+      // Workaround for Safari where the video frame renders black on
+      // unknown events. Trigger re-render frame on focus.
+      if (!isPlaying) {
+        bridge.goToFrame(bridge.frame);
+      }
+    }
+
+    function onVisibilityChange() {
+      // Workaround for Safari where the video frame renders black on
+      // visibility change hidden. Returning to visible shows a black
+      // frame instead of rendering the current frame.
+      if (document.visibilityState === 'visible' && !isPlaying) {
+        bridge.goToFrame(bridge.frame);
+      }
+    }
+
+    function onError(event: ErrorEvent) {
+      const error = event.error;
+      Logger.error(error);
+      reportError(error);
+    }
+
+    function onPlay() {
+      isPlaying = true;
+      setIsPlaying(true);
+    }
+    function onPause() {
+      isPlaying = false;
+      setIsPlaying(false);
+    }
+
+    function onStreamingDone(event: StreamingStateUpdateEvent) {
+      // continue to play after streaming is done (state is "full")
+      if (event.state === 'full') {
+        bridge.play();
+      }
+    }
+
+    function onLoadStart() {
+      setIsVideoLoading(true);
+    }
+
+    function onDecodeStart() {
+      setIsVideoLoading(false);
+    }
+
+    window.addEventListener('focus', onFocus);
+    window.addEventListener('visibilitychange', onVisibilityChange);
+    bridge.addEventListener('error', onError);
+    bridge.addEventListener('play', onPlay);
+    bridge.addEventListener('pause', onPause);
+    bridge.addEventListener('streamingStateUpdate', onStreamingDone);
+    bridge.addEventListener('loadstart', onLoadStart);
+    bridge.addEventListener('decode', onDecodeStart);
+    return () => {
+      window.removeEventListener('focus', onFocus);
+      window.removeEventListener('visibilitychange', onVisibilityChange);
+      bridge.removeEventListener('error', onError);
+      bridge.removeEventListener('play', onPlay);
+      bridge.removeEventListener('pause', onPause);
+      bridge.removeEventListener('streamingStateUpdate', onStreamingDone);
+      bridge.removeEventListener('loadstart', onLoadStart);
+      bridge.removeEventListener('decode', onDecodeStart);
+    };
+  }, [bridge, reportError, setIsPlaying, setIsVideoLoading]);
+
+  return (
+    <div
+      {...stylex.props(containerStyle ?? styles.container)}
+      ref={resizeObserverRef}>
+      <div {...stylex.props(styles.canvasContainer)}>
+        {(isVideoLoading || loading) && <VideoLoadingOverlay />}
+        <canvas
+          ref={canvasRef}
+          {...stylex.props(canvasStyle)}
+          className="lg:rounded-[4px]"
+          width={width}
+          height={height}
+          style={{
+            height: canvasHeight,
+          }}
+        />
+      </div>
+      {controls && (
+        <div {...stylex.props(styles.controls)}>
+          <Button
+            color="ghost"
+            size="xs"
+            startIcon={
+              isPlaying ? (
+                <PauseFilled
+                  {...stylex.props(styles.controlButton)}
+                  size={14}
+                />
+              ) : (
+                <PlayFilledAlt
+                  {...stylex.props(styles.controlButton)}
+                  size={14}
+                />
+              )
+            }
+            onClick={() => {
+              isPlaying ? bridge.pause() : bridge.play();
+            }}
+          />
+        </div>
+      )}
+    </div>
+  );
+});

sam2/demo/frontend/src/common/components/video/VideoFilmstripWithPlayback.tsx

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PlaybackButton from '@/common/components/button/PlaybackButton';
+import VideoFilmstrip from '@/common/components/video/filmstrip/VideoFilmstrip';
+import {spacing, w} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'end',
+    gap: spacing[4],
+    paddingHorizontal: spacing[4],
+    width: '100%',
+  },
+  playbackButtonContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    width: w[12],
+    height: w[12],
+  },
+  filmstripContainer: {
+    flexGrow: 1,
+  },
+});
+
+export default function VideoFilmstripWithPlayback() {
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.playbackButtonContainer)}>
+        <PlaybackButton />
+      </div>
+      <div {...stylex.props(styles.filmstripContainer)}>
+        <VideoFilmstrip />
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/video/VideoLoadingOverlay.tsx

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {fontSize, fontWeight, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {Loading} from 'react-daisyui';
+
+const styles = stylex.create({
+  overlay: {
+    position: 'absolute',
+    width: '100%',
+    height: '100%',
+    background: 'rgba(0,0,0,0.5)',
+  },
+  indicatorContainer: {
+    position: 'absolute',
+    top: '50%',
+    left: '50%',
+    transform: 'translate(-50%, -50%)',
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[4],
+    color: 'white',
+  },
+  indicatorText: {
+    color: 'white',
+    fontSize: fontSize['sm'],
+    fontWeight: fontWeight['medium'],
+  },
+});
+
+type Props = {
+  label?: string;
+};
+
+export default function VideoLoadingOverlay({label}: Props) {
+  return (
+    <div {...stylex.props(styles.overlay)}>
+      <div {...stylex.props(styles.indicatorContainer)}>
+        <Loading size="sm" />
+        <div {...stylex.props(styles.indicatorText)}>
+          {label ?? 'Loading video...'}
+        </div>
+      </div>
+    </div>
+  );
+}

sam2/demo/frontend/src/common/components/video/VideoWorker.ts

@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {Tracker} from '@/common/tracker/Tracker';
+import {TrackerRequestMessageEvent} from '@/common/tracker/TrackerTypes';
+import {TRACKER_MAPPING} from '@/common/tracker/Trackers';
+import {serializeError} from 'serialize-error';
+import VideoWorkerContext from './VideoWorkerContext';
+import {
+  ErrorResponse,
+  VideoWorkerRequestMessageEvent,
+} from './VideoWorkerTypes';
+
+registerSerializableConstructors();
+
+const context = new VideoWorkerContext();
+let tracker: Tracker | null = null;
+
+let statsEnabled = false;
+
+self.addEventListener(
+  'message',
+  async (
+    event: VideoWorkerRequestMessageEvent | TrackerRequestMessageEvent,
+  ) => {
+    try {
+      switch (event.data.action) {
+        // Initialize context
+        case 'setCanvas':
+          context.setCanvas(event.data.canvas);
+          break;
+        case 'setSource':
+          context.setSource(event.data.source);
+          break;
+
+        // Playback
+        case 'play':
+          context.play();
+          break;
+        case 'pause':
+          context.pause();
+          break;
+        case 'stop':
+          context.stop();
+          break;
+        case 'frameUpdate':
+          context.goToFrame(event.data.index);
+          break;
+
+        // Filmstrip
+        case 'filmstrip': {
+          const {width, height} = event.data;
+          await context.createFilmstrip(width, height);
+          break;
+        }
+
+        // Effects
+        case 'setEffect': {
+          const {name, index, options} = event.data;
+          await context.setEffect(name, index, options);
+          break;
+        }
+
+        // Encode
+        case 'encode': {
+          await context.encode();
+          break;
+        }
+
+        case 'enableStats': {
+          statsEnabled = true;
+          context.enableStats();
+          tracker?.enableStats();
+          break;
+        }
+
+        // Tracker
+        case 'initializeTracker': {
+          const {name, options} = event.data;
+          const Tracker = TRACKER_MAPPING[name];
+          // Update the endpoint for the streaming API
+          tracker = new Tracker(context, options);
+          if (statsEnabled) {
+            tracker.enableStats();
+          }
+          break;
+        }
+        case 'startSession': {
+          const {videoUrl} = event.data;
+          await tracker?.startSession(videoUrl);
+          break;
+        }
+        case 'createTracklet':
+          tracker?.createTracklet();
+          break;
+        case 'deleteTracklet':
+          await tracker?.deleteTracklet(event.data.trackletId);
+          break;
+        case 'closeSession':
+          tracker?.closeSession();
+          break;
+        case 'updatePoints': {
+          const {frameIndex, objectId, points} = event.data;
+          context.allowEffectAnimation(true, objectId, points);
+          await tracker?.updatePoints(frameIndex, objectId, points);
+          break;
+        }
+        case 'clearPointsInFrame': {
+          const {frameIndex, objectId} = event.data;
+          await tracker?.clearPointsInFrame(frameIndex, objectId);
+          break;
+        }
+        case 'clearPointsInVideo':
+          await tracker?.clearPointsInVideo();
+          break;
+        case 'streamMasks': {
+          const {frameIndex} = event.data;
+          context.allowEffectAnimation(false);
+          await tracker?.streamMasks(frameIndex);
+          break;
+        }
+        case 'abortStreamMasks':
+          tracker?.abortStreamMasks();
+          break;
+      }
+    } catch (error) {
+      const serializedError = serializeError(error);
+      const errorResponse: ErrorResponse = {
+        action: 'error',
+        error: serializedError,
+      };
+      self.postMessage(errorResponse);
+    }
+  },
+);

sam2/demo/frontend/src/common/components/video/VideoWorkerBridge.ts

@@ -0,0 +1,525 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {EffectIndex, Effects} from '@/common/components/video/effects/Effects';
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {
+  BaseTracklet,
+  SegmentationPoint,
+  StreamingState,
+} from '@/common/tracker/Tracker';
+import {
+  AbortStreamMasksRequest,
+  AddPointsResponse,
+  ClearPointsInFrameRequest,
+  ClearPointsInVideoRequest,
+  ClearPointsInVideoResponse,
+  CloseSessionRequest,
+  CreateTrackletRequest,
+  DeleteTrackletRequest,
+  InitializeTrackerRequest,
+  LogAnnotationsRequest,
+  SessionStartFailedResponse,
+  SessionStartedResponse,
+  StartSessionRequest,
+  StreamMasksRequest,
+  StreamingStateUpdateResponse,
+  TrackerRequest,
+  TrackerResponseMessageEvent,
+  TrackletCreatedResponse,
+  TrackletDeletedResponse,
+  UpdatePointsRequest,
+} from '@/common/tracker/TrackerTypes';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {MP4ArrayBuffer} from 'mp4box';
+import {deserializeError, type ErrorObject} from 'serialize-error';
+import {EventEmitter} from './EventEmitter';
+import {
+  EncodeVideoRequest,
+  FilmstripRequest,
+  FilmstripResponse,
+  FrameUpdateRequest,
+  PauseRequest,
+  PlayRequest,
+  SetCanvasRequest,
+  SetEffectRequest,
+  SetSourceRequest,
+  StopRequest,
+  VideoWorkerRequest,
+  VideoWorkerResponseMessageEvent,
+} from './VideoWorkerTypes';
+import {EffectOptions} from './effects/Effect';
+
+registerSerializableConstructors();
+
+export type DecodeEvent = {
+  totalFrames: number;
+  numFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+  done: boolean;
+};
+
+export type LoadStartEvent = unknown;
+
+export type EffectUpdateEvent = {
+  name: keyof Effects;
+  index: EffectIndex;
+  variant: number;
+  numVariants: number;
+};
+
+export type EncodingStateUpdateEvent = {
+  progress: number;
+};
+
+export type EncodingCompletedEvent = {
+  file: MP4ArrayBuffer;
+};
+
+export interface PlayEvent {}
+
+export interface PauseEvent {}
+
+export interface FilmstripEvent {
+  filmstrip: ImageBitmap;
+}
+
+export interface FrameUpdateEvent {
+  index: number;
+}
+
+export interface SessionStartedEvent {
+  sessionId: string;
+}
+
+export interface SessionStartFailedEvent {}
+
+export interface TrackletCreatedEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklet: BaseTracklet;
+}
+
+export interface TrackletsEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklets: BaseTracklet[];
+}
+
+export interface TrackletDeletedEvent {
+  isSuccessful: boolean;
+}
+
+export interface AddPointsEvent {
+  isSuccessful: boolean;
+}
+
+export interface ClearPointsInVideoEvent {
+  isSuccessful: boolean;
+}
+
+export interface StreamingStartedEvent {}
+
+export interface StreamingCompletedEvent {}
+
+export interface StreamingStateUpdateEvent {
+  state: StreamingState;
+}
+
+export interface RenderingErrorEvent {
+  error: ErrorObject;
+}
+
+export interface VideoWorkerEventMap {
+  error: ErrorEvent;
+  decode: DecodeEvent;
+  encodingStateUpdate: EncodingStateUpdateEvent;
+  encodingCompleted: EncodingCompletedEvent;
+  play: PlayEvent;
+  pause: PauseEvent;
+  filmstrip: FilmstripEvent;
+  frameUpdate: FrameUpdateEvent;
+  sessionStarted: SessionStartedEvent;
+  sessionStartFailed: SessionStartFailedEvent;
+  trackletCreated: TrackletCreatedEvent;
+  trackletsUpdated: TrackletsEvent;
+  trackletDeleted: TrackletDeletedEvent;
+  addPoints: AddPointsEvent;
+  clearPointsInVideo: ClearPointsInVideoEvent;
+  streamingStarted: StreamingStartedEvent;
+  streamingCompleted: StreamingCompletedEvent;
+  streamingStateUpdate: StreamingStateUpdateEvent;
+  // HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+  loadstart: LoadStartEvent;
+  effectUpdate: EffectUpdateEvent;
+  renderingError: RenderingErrorEvent;
+}
+
+type Metadata = {
+  totalFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+};
+
+export default class VideoWorkerBridge extends EventEmitter<VideoWorkerEventMap> {
+  static create(workerFactory: () => Worker) {
+    const worker = workerFactory();
+    return new VideoWorkerBridge(worker);
+  }
+
+  protected worker: Worker;
+  private metadata: Metadata | null = null;
+  private frameIndex: number = 0;
+
+  private _sessionId: string | null = null;
+
+  public get sessionId() {
+    return this._sessionId;
+  }
+
+  public get width() {
+    return this.metadata?.width ?? 0;
+  }
+
+  public get height() {
+    return this.metadata?.height ?? 0;
+  }
+
+  public get numberOfFrames() {
+    return this.metadata?.totalFrames ?? 0;
+  }
+
+  public get fps() {
+    return this.metadata?.fps ?? 0;
+  }
+
+  public get frame() {
+    return this.frameIndex;
+  }
+
+  constructor(worker: Worker) {
+    super();
+    this.worker = worker;
+
+    worker.addEventListener(
+      'message',
+      (
+        event: VideoWorkerResponseMessageEvent | TrackerResponseMessageEvent,
+      ) => {
+        switch (event.data.action) {
+          case 'error':
+            // Deserialize error before triggering the event
+            event.data.error = deserializeError(event.data.error);
+            break;
+          case 'decode':
+            this.metadata = event.data;
+            break;
+          case 'frameUpdate':
+            this.frameIndex = event.data.index;
+            break;
+          case 'sessionStarted':
+            this._sessionId = event.data.sessionId;
+            break;
+        }
+        this.trigger(event.data.action, event.data);
+      },
+    );
+  }
+
+  public setCanvas(canvas: HTMLCanvasElement): void {
+    const offscreenCanvas = canvas.transferControlToOffscreen();
+    this.sendRequest<SetCanvasRequest>(
+      'setCanvas',
+      {
+        canvas: offscreenCanvas,
+      },
+      [offscreenCanvas],
+    );
+  }
+
+  public setSource(source: string): void {
+    this.sendRequest<SetSourceRequest>('setSource', {
+      source,
+    });
+  }
+
+  public terminate(): void {
+    super.destroy();
+    this.worker.terminate();
+  }
+
+  public play(): void {
+    this.sendRequest<PlayRequest>('play');
+  }
+
+  public pause(): void {
+    this.sendRequest<PauseRequest>('pause');
+  }
+
+  public stop(): void {
+    this.sendRequest<StopRequest>('stop');
+  }
+
+  public goToFrame(index: number): void {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {
+      index,
+    });
+  }
+
+  public previousFrame(): void {
+    const index = Math.max(0, this.frameIndex - 1);
+    this.goToFrame(index);
+  }
+
+  public nextFrame(): void {
+    const index = Math.min(this.frameIndex + 1, this.numberOfFrames - 1);
+    this.goToFrame(index);
+  }
+
+  public set frame(index: number) {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {index});
+  }
+
+  createFilmstrip(width: number, height: number): Promise<ImageBitmap> {
+    return new Promise((resolve, _reject) => {
+      const handleFilmstripResponse = (
+        event: MessageEvent<FilmstripResponse>,
+      ) => {
+        if (event.data.action === 'filmstrip') {
+          this.worker.removeEventListener('message', handleFilmstripResponse);
+          resolve(event.data.filmstrip);
+        }
+      };
+
+      this.worker.addEventListener('message', handleFilmstripResponse);
+
+      this.sendRequest<FilmstripRequest>('filmstrip', {
+        width,
+        height,
+      });
+    });
+  }
+
+  setEffect(name: keyof Effects, index: EffectIndex, options?: EffectOptions) {
+    this.sendRequest<SetEffectRequest>('setEffect', {
+      name,
+      index,
+      options,
+    });
+  }
+
+  encode(): void {
+    this.sendRequest<EncodeVideoRequest>('encode');
+  }
+
+  initializeTracker(name: keyof Trackers, options: TrackerOptions): void {
+    this.sendRequest<InitializeTrackerRequest>('initializeTracker', {
+      name,
+      options,
+    });
+  }
+
+  startSession(videoUrl: string): Promise<string | null> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<
+          SessionStartedResponse | SessionStartFailedResponse
+        >,
+      ) => {
+        if (event.data.action === 'sessionStarted') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.sessionId);
+        }
+        if (event.data.action === 'sessionStartFailed') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(null);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<StartSessionRequest>('startSession', {
+        videoUrl,
+      });
+    });
+  }
+
+  closeSession(): void {
+    this.sendRequest<CloseSessionRequest>('closeSession');
+  }
+
+  logAnnotations(): void {
+    this.sendRequest<LogAnnotationsRequest>('logAnnotations');
+  }
+
+  createTracklet(): Promise<BaseTracklet> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<TrackletCreatedResponse>) => {
+        if (event.data.action === 'trackletCreated') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.tracklet);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<CreateTrackletRequest>('createTracklet');
+    });
+  }
+
+  deleteTracklet(trackletId: number): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const handleResponse = (event: MessageEvent<TrackletDeletedResponse>) => {
+        if (event.data.action === 'trackletDeleted') {
+          this.worker.removeEventListener('message', handleResponse);
+          if (event.data.isSuccessful) {
+            resolve();
+          } else {
+            reject(`could not delete tracklet ${trackletId}`);
+          }
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<DeleteTrackletRequest>('deleteTracklet', {trackletId});
+    });
+  }
+
+  updatePoints(
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<AddPointsResponse>) => {
+        if (event.data.action === 'addPoints') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<UpdatePointsRequest>('updatePoints', {
+        frameIndex: this.frame,
+        objectId,
+        points,
+      });
+    });
+  }
+
+  clearPointsInFrame(objectId: number) {
+    this.sendRequest<ClearPointsInFrameRequest>('clearPointsInFrame', {
+      frameIndex: this.frame,
+      objectId,
+    });
+  }
+
+  clearPointsInVideo(): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<ClearPointsInVideoResponse>,
+      ) => {
+        if (event.data.action === 'clearPointsInVideo') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<ClearPointsInVideoRequest>('clearPointsInVideo');
+    });
+  }
+
+  streamMasks(): void {
+    this.sendRequest<StreamMasksRequest>('streamMasks', {
+      frameIndex: this.frame,
+    });
+  }
+
+  abortStreamMasks(): Promise<void> {
+    return new Promise(resolve => {
+      const handleAbortResponse = (
+        event: MessageEvent<StreamingStateUpdateResponse>,
+      ) => {
+        if (
+          event.data.action === 'streamingStateUpdate' &&
+          event.data.state === 'aborted'
+        ) {
+          this.worker.removeEventListener('message', handleAbortResponse);
+          resolve();
+        }
+      };
+
+      this.worker.addEventListener('message', handleAbortResponse);
+      this.sendRequest<AbortStreamMasksRequest>('abortStreamMasks');
+    });
+  }
+
+  getWorker_ONLY_USE_WITH_CAUTION(): Worker {
+    return this.worker;
+  }
+
+  /**
+   * Convenient function to have typed postMessage.
+   *
+   * @param action Video worker action
+   * @param message Actual payload
+   * @param transfer Any object that should be transferred instead of cloned
+   */
+  protected sendRequest<T extends VideoWorkerRequest | TrackerRequest>(
+    action: T['action'],
+    payload?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ) {
+    this.worker.postMessage(
+      {
+        action,
+        ...payload,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  // // Override EventEmitter
+
+  // addEventListener<K extends keyof WorkerEventMap>(
+  //   type: K,
+  //   listener: (ev: WorkerEventMap[K]) => unknown,
+  // ): void {
+  //   switch (type) {
+  //     case 'frameUpdate':
+  //       {
+  //         const event: FrameUpdateEvent = {
+  //           index: this.frameIndex,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //       break;
+  //     case 'sessionStarted': {
+  //       if (this.sessionId !== null) {
+  //         const event: SessionStartedEvent = {
+  //           sessionId: this.sessionId,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //     }
+  //   }
+  //   super.addEventListener(type, listener);
+  // }
+}

sam2/demo/frontend/src/common/components/video/VideoWorkerContext.ts

@@ -0,0 +1,843 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodedVideo,
+  ImageFrame,
+  decodeStream,
+} from '@/common/codecs/VideoDecoder';
+import {encode as encodeVideo} from '@/common/codecs/VideoEncoder';
+import {
+  Effect,
+  EffectActionPoint,
+  EffectFrameContext,
+  EffectOptions,
+} from '@/common/components/video/effects/Effect';
+import AllEffects, {
+  EffectIndex,
+  Effects,
+} from '@/common/components/video/effects/Effects';
+import Logger from '@/common/logger/Logger';
+import {Mask, SegmentationPoint, Tracklet} from '@/common/tracker/Tracker';
+import {streamFile} from '@/common/utils/FileUtils';
+import {Stats} from '@/debug/stats/Stats';
+import {VIDEO_WATERMARK_TEXT} from '@/demo/DemoConfig';
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {RLEObject} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+import {serializeError} from 'serialize-error';
+import {
+  DecodeResponse,
+  EffectUpdateResponse,
+  EncodingCompletedResponse,
+  EncodingStateUpdateResponse,
+  FilmstripResponse,
+  FrameUpdateResponse,
+  PauseRequest,
+  PlayRequest,
+  RenderingErrorResponse,
+  VideoWorkerResponse,
+} from './VideoWorkerTypes';
+
+function getEvenlySpacedItems(decodedVideo: DecodedVideo, x: number) {
+  const p = Math.floor(decodedVideo.numFrames / Math.max(1, x - 1));
+  const middleFrames = decodedVideo.frames
+    .slice(p, p * x)
+    .filter(function (_, i) {
+      return 0 == i % p;
+    });
+  return [
+    decodedVideo.frames[0],
+    ...middleFrames,
+    decodedVideo.frames[decodedVideo.numFrames - 1],
+  ];
+}
+
+export type FrameInfo = {
+  tracklet: Tracklet;
+  mask: Mask;
+};
+
+const WATERMARK_BOX_HORIZONTAL_PADDING = 10;
+const WATERMARK_BOX_VERTICAL_PADDING = 10;
+
+export type VideoStats = {
+  fps?: Stats;
+  videoFps?: Stats;
+  total?: Stats;
+  effect0?: Stats;
+  effect1?: Stats;
+  frameBmp?: Stats;
+  maskBmp?: Stats;
+  memory?: Stats;
+};
+
+export default class VideoWorkerContext {
+  private _canvas: OffscreenCanvas | null = null;
+  private _stats: VideoStats = {};
+  private _ctx: OffscreenCanvasRenderingContext2D | null = null;
+  private _form: CanvasForm | null = null;
+  private _decodedVideo: DecodedVideo | null = null;
+  private _frameIndex: number = 0;
+  private _isPlaying: boolean = false;
+  private _playbackRAFHandle: number | null = null;
+  private _playbackTimeoutHandle: NodeJS.Timeout | null = null;
+  private _isDrawing: boolean = false;
+  private _glObjects: WebGL2RenderingContext | null = null;
+  private _glBackground: WebGL2RenderingContext | null = null;
+  private _canvasHighlights: OffscreenCanvas | null = null;
+  private _canvasBackground: OffscreenCanvas | null = null;
+  private _allowAnimation: boolean = false;
+  private _currentSegmetationPoint: EffectActionPoint | null = null;
+
+  private _effects: Effect[];
+  private _tracklets: Tracklet[] = [];
+
+  public get width(): number {
+    return this._decodedVideo?.width ?? 0;
+  }
+
+  public get height(): number {
+    return this._decodedVideo?.height ?? 0;
+  }
+
+  public get frameIndex(): number {
+    return this._frameIndex;
+  }
+
+  public get currentFrame(): VideoFrame | null {
+    return this._decodedVideo?.frames[this._frameIndex].bitmap ?? null;
+  }
+
+  constructor() {
+    this._effects = [
+      AllEffects.Original, // Image as background
+      AllEffects.Overlay, // Masks on top
+    ];
+
+    // Loading watermark fonts. This is going to be async, but by the time of
+    // video encoding, the fonts should be available.
+    this._loadWatermarkFonts();
+  }
+
+  private initializeWebGLContext(width: number, height: number): void {
+    // Given that we use highlight and background effects as layers,
+    // we need to create two WebGL contexts, one for each set.
+    // To avoid memory leaks and too many active contexts,
+    // these contexts must be re-used over the lifecycle of the session.
+
+    if (this._canvasHighlights == null && this._glObjects == null) {
+      this._canvasHighlights = new OffscreenCanvas(width, height);
+      this._glObjects = this._canvasHighlights.getContext('webgl2');
+
+      this._canvasHighlights.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasHighlights != null &&
+      (this._canvasHighlights.width !== width ||
+        this._canvasHighlights.height !== height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasHighlights.width = width;
+      this._canvasHighlights.height = height;
+      if (this._glObjects != null) {
+        this._glObjects.viewport(0, 0, width, height);
+      }
+    }
+
+    if (this._canvasBackground == null && this._glBackground == null) {
+      this._canvasBackground = new OffscreenCanvas(width, height);
+      this._glBackground = this._canvasBackground.getContext('webgl2');
+
+      this._canvasBackground.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasBackground != null &&
+      (this._canvasBackground.width != width ||
+        this._canvasBackground.height != height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasBackground.width = width;
+      this._canvasBackground.height = height;
+      if (this._glBackground != null) {
+        this._glBackground.viewport(0, 0, width, height);
+      }
+    }
+  }
+
+  public setCanvas(canvas: OffscreenCanvas) {
+    this._canvas = canvas;
+    this._ctx = canvas.getContext('2d');
+    if (this._ctx == null) {
+      throw new Error('could not initialize drawing context');
+    }
+    this._form = new CanvasForm(this._ctx);
+  }
+
+  public setSource(src: string) {
+    this.close();
+
+    // Clear state of previous source.
+    this.updateFrameIndex(0);
+    this._tracklets = [];
+
+    this._decodeVideo(src);
+  }
+
+  public goToFrame(index: number): void {
+    // Cancel any ongoing render
+    this._cancelRender();
+    this.updateFrameIndex(index);
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  public play(): void {
+    // Video already playing
+    if (this._isPlaying) {
+      return;
+    }
+
+    // Cannot playback without frames
+    if (this._decodedVideo === null) {
+      throw new Error('no decoded video');
+    }
+
+    const {numFrames, fps} = this._decodedVideo;
+    const timePerFrame = 1000 / (fps ?? 30);
+    let startTime: number | null = null;
+    // The offset frame index compensate for cases where the video playback
+    // does not start at frame index 0.
+    const offsetFrameIndex = this._frameIndex;
+
+    const updateFrame = (time: number) => {
+      if (startTime === null) {
+        startTime = time;
+      }
+
+      this._stats.fps?.begin();
+
+      const diff = time - startTime;
+      const expectedFrame =
+        (Math.floor(diff / timePerFrame) + offsetFrameIndex) % numFrames;
+
+      if (this._frameIndex !== expectedFrame && !this._isDrawing) {
+        // Update to the next expected frame
+        this.updateFrameIndex(expectedFrame);
+        this._drawFrame();
+      }
+
+      this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+
+      this._stats.fps?.end();
+    };
+
+    this.updatePlayback(true);
+    this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+  }
+
+  public pause(): void {
+    this.updatePlayback(false);
+    this._cancelRender();
+  }
+
+  public stop(): void {
+    this.pause();
+    this.updateFrameIndex(0);
+  }
+
+  public async createFilmstrip(width: number, height: number): Promise<void> {
+    if (width < 1 || height < 1) {
+      Logger.warn(
+        `Cannot create filmstrip because width ${width} or height ${height} is too small.`,
+      );
+      return;
+    }
+
+    try {
+      const canvas = new OffscreenCanvas(width, height);
+      const ctx = canvas.getContext('2d');
+
+      if (this._decodedVideo !== null) {
+        const scale = canvas.height / this._decodedVideo.height;
+        const resizeWidth = this._decodedVideo.width * scale;
+
+        const spacedFrames = getEvenlySpacedItems(
+          this._decodedVideo,
+          Math.ceil(canvas.width / resizeWidth),
+        );
+
+        spacedFrames.forEach((frame, idx) => {
+          if (frame != null) {
+            ctx?.drawImage(
+              frame.bitmap,
+              resizeWidth * idx,
+              0,
+              resizeWidth,
+              canvas.height,
+            );
+          }
+        });
+      }
+
+      const filmstrip = await createImageBitmap(canvas);
+
+      this.sendResponse<FilmstripResponse>(
+        'filmstrip',
+        {
+          filmstrip,
+        },
+        [filmstrip],
+      );
+    } catch {
+      this._sendRenderingError(
+        new CreateFilmstripError('Failed to create filmstrip'),
+      );
+    }
+  }
+
+  public async setEffect(
+    name: keyof Effects,
+    index: EffectIndex,
+    options?: EffectOptions,
+  ): Promise<void> {
+    const effect: Effect = AllEffects[name];
+
+    // The effect has changed.
+    if (this._effects[index] !== effect) {
+      // Effect changed. Cleanup old effect first. Effects are responsible for
+      // cleaning up their memory.
+      await this._effects[index].cleanup();
+
+      const offCanvas =
+        index === EffectIndex.BACKGROUND
+          ? this._canvasBackground
+          : this._canvasHighlights;
+      invariant(offCanvas != null, 'need OffscreenCanvas to render effects');
+      const webglContext =
+        index === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+      invariant(webglContext != null, 'need WebGL context to render effects');
+
+      // Initialize the effect. This can be used by effects to prepare
+      // resources needed for rendering. If the video wasn't decoded yet, the
+      // effect setup will happen in the _decodeVideo function.
+      if (this._decodedVideo != null) {
+        await effect.setup({
+          width: this._decodedVideo.width,
+          height: this._decodedVideo.height,
+          canvas: offCanvas,
+          gl: webglContext,
+        });
+      }
+    }
+
+    // Update effect if already set effect was clicked again. This can happen
+    // when there is a new variant of the effect.
+    if (options != null) {
+      // Update effect if already set effect was clicked again. This can happen
+      // when there is a new variant of the effect.
+      await effect.update(options);
+    }
+
+    // Notify the frontend about the effect state including its variant.
+    this.sendResponse<EffectUpdateResponse>('effectUpdate', {
+      name,
+      index,
+      variant: effect.variant,
+      numVariants: effect.numVariants,
+    });
+
+    this._effects[index] = effect;
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  async encode() {
+    const decodedVideo = this._decodedVideo;
+    invariant(
+      decodedVideo !== null,
+      'cannot encode video because there is no decoded video available',
+    );
+
+    const canvas = new OffscreenCanvas(this.width, this.height);
+    const ctx = canvas.getContext('2d', {willReadFrequently: true});
+    invariant(
+      ctx !== null,
+      'cannot encode video because failed to construct offscreen canvas context',
+    );
+
+    const form = new CanvasForm(ctx);
+
+    const file = await encodeVideo(
+      this.width,
+      this.height,
+      decodedVideo.frames.length,
+      this._framesGenerator(decodedVideo, canvas, form),
+      progress => {
+        this.sendResponse<EncodingStateUpdateResponse>('encodingStateUpdate', {
+          progress,
+        });
+      },
+    );
+    this.sendResponse<EncodingCompletedResponse>(
+      'encodingCompleted',
+      {
+        file,
+      },
+      [file],
+    );
+  }
+
+  private async *_framesGenerator(
+    decodedVideo: DecodedVideo,
+    canvas: OffscreenCanvas,
+    form: CanvasForm,
+  ): AsyncGenerator<ImageFrame, undefined> {
+    const frames = decodedVideo.frames;
+
+    for (let frameIndex = 0; frameIndex < frames.length; ++frameIndex) {
+      await this._drawFrameImpl(form, frameIndex, true);
+
+      const frame = frames[frameIndex];
+      const videoFrame = new VideoFrame(canvas, {
+        timestamp: frame.bitmap.timestamp,
+      });
+
+      yield {
+        bitmap: videoFrame,
+        timestamp: frame.timestamp,
+        duration: frame.duration,
+      };
+
+      videoFrame.close();
+    }
+  }
+
+  public enableStats() {
+    this._stats.fps = new Stats('fps');
+    this._stats.videoFps = new Stats('fps', 'V');
+    this._stats.total = new Stats('ms', 'T');
+    this._stats.effect0 = new Stats('ms', 'B');
+    this._stats.effect1 = new Stats('ms', 'H');
+    this._stats.frameBmp = new Stats('ms', 'F');
+    this._stats.maskBmp = new Stats('ms', 'M');
+    this._stats.memory = new Stats('memory');
+  }
+
+  public allowEffectAnimation(
+    allow: boolean = true,
+    objectId?: number,
+    points?: SegmentationPoint[],
+  ) {
+    if (objectId != null && points != null && points.length) {
+      const last_point_position = points[points.length - 1];
+      this._currentSegmetationPoint = {
+        objectId,
+        position: [last_point_position[0], last_point_position[1]],
+      };
+    }
+
+    if (!allow) {
+      this._currentSegmetationPoint = null;
+    }
+
+    this._allowAnimation = allow;
+  }
+
+  public close(): void {
+    // Clear any frame content
+    this._ctx?.reset();
+
+    // Close frames of previously decoded video.
+    this._decodedVideo?.frames.forEach(f => f.bitmap.close());
+    this._decodedVideo = null;
+  }
+
+  // TRACKER
+
+  public updateTracklets(
+    frameIndex: number,
+    tracklets: Tracklet[],
+    shouldGoToFrame: boolean = true,
+  ): void {
+    this._tracklets = tracklets;
+    if (shouldGoToFrame) {
+      this.goToFrame(frameIndex);
+    }
+  }
+
+  public clearTrackletMasks(tracklet: Tracklet): void {
+    this._tracklets = this._tracklets.filter(t => t.id != tracklet.id);
+  }
+
+  public clearMasks(): void {
+    this._tracklets = [];
+  }
+
+  // PRIVATE FUNCTIONS
+
+  private sendResponse<T extends VideoWorkerResponse>(
+    action: T['action'],
+    message?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ): void {
+    self.postMessage(
+      {
+        action,
+        ...message,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  private async _decodeVideo(src: string): Promise<void> {
+    const canvas = this._canvas;
+    invariant(canvas != null, 'need canvas to render decoded video');
+
+    this.sendResponse('loadstart');
+
+    const fileStream = streamFile(src, {
+      credentials: 'same-origin',
+      cache: 'no-store',
+    });
+
+    let renderedFirstFrame = false;
+    this._decodedVideo = await decodeStream(fileStream, async progress => {
+      const {fps, height, width, numFrames, frames} = progress;
+      this._decodedVideo = progress;
+      if (!renderedFirstFrame) {
+        renderedFirstFrame = true;
+        canvas.width = width;
+        canvas.height = height;
+        // Set WebGL contexts right after the first frame decoded
+        this.initializeWebGLContext(width, height);
+
+        // Initialize effect once first frame was decoded.
+        for (const [i, effect] of this._effects.entries()) {
+          const offCanvas =
+            i === EffectIndex.BACKGROUND
+              ? this._canvasBackground
+              : this._canvasHighlights;
+          invariant(offCanvas != null, 'need canvas to render effects');
+          const webglContext =
+            i === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+          invariant(
+            webglContext != null,
+            'need WebGL context to render effects',
+          );
+          await effect.setup({
+            width,
+            height,
+            canvas: offCanvas,
+            gl: webglContext,
+          });
+        }
+
+        // Need to render frame immediately. Cannot go through
+        // requestAnimationFrame because then rendering this frame would be
+        // delayed until the full video has finished decoding.
+        this._drawFrame();
+
+        this._stats.videoFps?.updateMaxValue(fps);
+        this._stats.total?.updateMaxValue(1000 / fps);
+        this._stats.effect0?.updateMaxValue(1000 / fps);
+        this._stats.effect1?.updateMaxValue(1000 / fps);
+        this._stats.frameBmp?.updateMaxValue(1000 / fps);
+        this._stats.maskBmp?.updateMaxValue(1000 / fps);
+      }
+      this.sendResponse<DecodeResponse>('decode', {
+        totalFrames: numFrames,
+        numFrames: frames.length,
+        fps: fps,
+        width: width,
+        height: height,
+        done: false,
+      });
+    });
+
+    if (!renderedFirstFrame) {
+      canvas.width = this._decodedVideo.width;
+      canvas.height = this._decodedVideo.height;
+      this._drawFrame();
+    }
+
+    this.sendResponse<DecodeResponse>('decode', {
+      totalFrames: this._decodedVideo.numFrames,
+      numFrames: this._decodedVideo.frames.length,
+      fps: this._decodedVideo.fps,
+      width: this._decodedVideo.width,
+      height: this._decodedVideo.height,
+      done: true,
+    });
+  }
+
+  private _drawFrame(): void {
+    if (this._canvas !== null && this._form !== null) {
+      this._drawFrameImpl(this._form, this._frameIndex);
+    }
+  }
+
+  private async _drawFrameImpl(
+    form: CanvasForm,
+    frameIndex: number,
+    enableWatermark: boolean = false,
+    step: number = 0,
+    maxSteps: number = 40,
+  ): Promise<void> {
+    if (this._decodedVideo === null) {
+      return;
+    }
+
+    {
+      this._stats.videoFps?.begin();
+      this._stats.total?.begin();
+      this._stats.memory?.begin();
+    }
+
+    try {
+      const frame = this._decodedVideo.frames[frameIndex];
+      const {bitmap} = frame;
+
+      this._stats.frameBmp?.begin();
+
+      // Need to convert VideoFrame to ImageBitmap because Safari can only apply
+      // globalCompositeOperation on ImageBitmap and fails on VideoFrame. FWIW,
+      // Chrome treats VideoFrame similarly to ImageBitmap.
+      const frameBitmap = await createImageBitmap(bitmap);
+
+      this._stats.frameBmp?.end();
+
+      const masks: Mask[] = [];
+      const colors: string[] = [];
+      const tracklets: Tracklet[] = [];
+      this._tracklets.forEach(tracklet => {
+        const mask = tracklet.masks[frameIndex];
+        if (mask != null) {
+          masks.push(mask);
+          tracklets.push(tracklet);
+          colors.push(tracklet.color);
+        }
+      });
+      const effectActionPoint = this._currentSegmetationPoint;
+
+      this._stats.maskBmp?.begin();
+
+      const effectMaskPromises = masks.map(async ({data, bounds}) => {
+        return {
+          bounds,
+          bitmap: data as RLEObject,
+        };
+      });
+      const effectMasks = await Promise.all(effectMaskPromises);
+
+      this._stats.maskBmp?.end();
+
+      form.ctx.fillStyle = 'rgba(0, 0, 0, 0)';
+      form.ctx.fillRect(0, 0, this.width, this.height);
+
+      const effectParams: EffectFrameContext = {
+        frame: frameBitmap,
+        masks: effectMasks,
+        maskColors: colors,
+        frameIndex: frameIndex,
+        totalFrames: this._decodedVideo.frames.length,
+        fps: this._decodedVideo.fps,
+        width: frameBitmap.width,
+        height: frameBitmap.height,
+        actionPoint: null,
+      };
+
+      // Allows animation within a single frame.
+      if (this._allowAnimation && step < maxSteps) {
+        const animationDuration = 2; // Total duration of the animation in seconds
+        const progress = step / maxSteps;
+        const timeParameter = progress * animationDuration;
+        // Pass dynamic effect params
+        effectParams.timeParameter = timeParameter;
+        effectParams.actionPoint = effectActionPoint;
+
+        this._processEffects(form, effectParams, tracklets);
+
+        // Use RAF to draw frame, and update the display,
+        // this avoids to wait until the javascript call stack is cleared.
+        requestAnimationFrame(() =>
+          this._drawFrameImpl(form, frameIndex, false, step + 1, maxSteps),
+        );
+      } else {
+        this._processEffects(form, effectParams, tracklets);
+      }
+
+      if (enableWatermark) {
+        this._drawWatermark(form, frameBitmap);
+      }
+
+      // Do not simply drop the JavaScript reference to the ImageBitmap; doing so
+      // will keep its graphics resource alive until the next time the garbage
+      // collector runs.
+      frameBitmap.close();
+
+      {
+        this._stats.videoFps?.end();
+        this._stats.total?.end();
+        this._stats.memory?.end();
+      }
+
+      this._isDrawing = false;
+    } catch {
+      this._sendRenderingError(new DrawFrameError('Failed to draw frame'));
+    }
+  }
+
+  private _drawWatermark(form: CanvasForm, frameBitmap: ImageBitmap): void {
+    const frameWidth = this._canvas?.width || frameBitmap.width;
+    const frameHeight = this._canvas?.height || frameBitmap.height;
+    // Font size is either 12 or smaller based on available width
+    // since the font is not monospaced, we approximate it'll fit 1.5 more characters than monospaced
+    const approximateFontSize = Math.min(
+      Math.floor(frameWidth / (VIDEO_WATERMARK_TEXT.length / 1.5)),
+      12,
+    );
+
+    form.ctx.font = `${approximateFontSize}px "Inter", sans-serif`;
+    const measureGeneratedBy = form.ctx.measureText(VIDEO_WATERMARK_TEXT);
+
+    const textBoxWidth =
+      measureGeneratedBy.width + 2 * WATERMARK_BOX_HORIZONTAL_PADDING;
+    const textBoxHeight =
+      measureGeneratedBy.actualBoundingBoxAscent +
+      2 * WATERMARK_BOX_VERTICAL_PADDING;
+    const textBoxX = frameWidth - textBoxWidth;
+    const textBoxY = frameHeight - textBoxHeight;
+
+    form.ctx.fillStyle = 'rgba(0, 0, 0, 0.4)';
+    form.ctx.beginPath();
+    form.ctx.roundRect(
+      Math.round(textBoxX),
+      Math.round(textBoxY),
+      Math.round(textBoxWidth),
+      Math.round(textBoxHeight),
+      [WATERMARK_BOX_HORIZONTAL_PADDING, 0, 0, 0],
+    );
+    form.ctx.fill();
+
+    // Always reset the text style because some effects may change text styling in the same ctx
+    form.ctx.fillStyle = 'rgba(255, 255, 255, 0.8)';
+    form.ctx.textAlign = 'left';
+
+    form.ctx.fillText(
+      VIDEO_WATERMARK_TEXT,
+      Math.round(textBoxX + WATERMARK_BOX_HORIZONTAL_PADDING),
+      Math.round(
+        textBoxY +
+          WATERMARK_BOX_VERTICAL_PADDING +
+          measureGeneratedBy.actualBoundingBoxAscent,
+      ),
+    );
+  }
+
+  private updateFrameIndex(index: number): void {
+    this._frameIndex = index;
+    this.sendResponse<FrameUpdateResponse>('frameUpdate', {
+      index,
+    });
+  }
+
+  private _loadWatermarkFonts() {
+    const requiredFonts = [
+      {
+        url: '/fonts/Inter-VariableFont.ttf',
+        format: 'truetype-variations',
+      },
+    ];
+    requiredFonts.forEach(requiredFont => {
+      const fontFace = new FontFace(
+        'Inter',
+        `url(${requiredFont.url}) format('${requiredFont.format}')`,
+      );
+      fontFace.load().then(font => {
+        self.fonts.add(font);
+      });
+    });
+  }
+
+  private updatePlayback(playing: boolean): void {
+    if (playing) {
+      this.sendResponse<PlayRequest>('play');
+    } else {
+      this.sendResponse<PauseRequest>('pause');
+    }
+    this._isPlaying = playing;
+  }
+
+  private _cancelRender(): void {
+    if (this._playbackTimeoutHandle !== null) {
+      clearTimeout(this._playbackTimeoutHandle);
+      this._playbackTimeoutHandle = null;
+    }
+    if (this._playbackRAFHandle !== null) {
+      cancelAnimationFrame(this._playbackRAFHandle);
+      this._playbackRAFHandle = null;
+    }
+  }
+
+  private _sendRenderingError(error: Error): void {
+    this.sendResponse<RenderingErrorResponse>('renderingError', {
+      error: serializeError(error),
+    });
+  }
+
+  private _processEffects(
+    form: CanvasForm,
+    effectParams: EffectFrameContext,
+    tracklets: Tracklet[],
+  ) {
+    for (let i = 0; i < this._effects.length; i++) {
+      const effect = this._effects[i];
+
+      if (i === 0) {
+        this._stats.effect0?.begin();
+      } else if (i === 1) {
+        this._stats.effect1?.begin();
+      }
+
+      effect.apply(form, effectParams, tracklets);
+
+      if (i === 0) {
+        this._stats.effect0?.end();
+      } else if (i === 1) {
+        this._stats.effect1?.end();
+      }
+    }
+  }
+}

sam2/demo/frontend/src/common/components/video/VideoWorkerTypes.ts

@@ -0,0 +1,143 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodeEvent,
+  EffectUpdateEvent,
+  EncodingCompletedEvent,
+  EncodingStateUpdateEvent,
+  FilmstripEvent,
+  FrameUpdateEvent,
+  LoadStartEvent,
+  RenderingErrorEvent,
+} from './VideoWorkerBridge';
+import {EffectOptions} from './effects/Effect';
+import type {Effects} from './effects/Effects';
+
+export type Request<A, P> = {
+  action: A;
+} & P;
+
+// REQUESTS
+
+export type SetCanvasRequest = Request<
+  'setCanvas',
+  {
+    canvas: OffscreenCanvas;
+  }
+>;
+export type SetSourceRequest = Request<
+  'setSource',
+  {
+    source: string;
+  }
+>;
+export type PlayRequest = Request<'play', unknown>;
+export type PauseRequest = Request<'pause', unknown>;
+export type StopRequest = Request<'stop', unknown>;
+export type FrameUpdateRequest = Request<
+  'frameUpdate',
+  {
+    index: number;
+  }
+>;
+export type FilmstripRequest = Request<
+  'filmstrip',
+  {
+    width: number;
+    height: number;
+  }
+>;
+export type SetEffectRequest = Request<
+  'setEffect',
+  {
+    name: keyof Effects;
+    index: number;
+    options?: EffectOptions;
+  }
+>;
+
+export type EncodeVideoRequest = Request<'encode', unknown>;
+
+export type EnableStatsRequest = Request<'enableStats', unknown>;
+
+export type VideoWorkerRequest =
+  | SetCanvasRequest
+  | SetSourceRequest
+  | PlayRequest
+  | PauseRequest
+  | StopRequest
+  | FrameUpdateRequest
+  | FilmstripRequest
+  | SetEffectRequest
+  | EncodeVideoRequest
+  | EnableStatsRequest;
+
+export type VideoWorkerRequestMessageEvent = MessageEvent<VideoWorkerRequest>;
+
+// RESPONSES
+
+export type ErrorResponse = Request<
+  'error',
+  {
+    error: unknown;
+  }
+>;
+
+export type DecodeResponse = Request<'decode', DecodeEvent>;
+
+export type EncodingStateUpdateResponse = Request<
+  'encodingStateUpdate',
+  EncodingStateUpdateEvent
+>;
+
+export type EncodingCompletedResponse = Request<
+  'encodingCompleted',
+  EncodingCompletedEvent
+>;
+
+export type FilmstripResponse = Request<'filmstrip', FilmstripEvent>;
+
+export type PlayResponse = Request<'play', unknown>;
+
+export type PauseResponse = Request<'pause', unknown>;
+
+export type FrameUpdateResponse = Request<'frameUpdate', FrameUpdateEvent>;
+
+export type RenderingErrorResponse = Request<
+  'renderingError',
+  RenderingErrorEvent
+>;
+
+// HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+
+export type LoadStartResponse = Request<'loadstart', LoadStartEvent>;
+
+export type EffectUpdateResponse = Request<'effectUpdate', EffectUpdateEvent>;
+
+export type VideoWorkerResponse =
+  | ErrorResponse
+  | FilmstripResponse
+  | DecodeResponse
+  | EncodingStateUpdateResponse
+  | EncodingCompletedResponse
+  | PlayResponse
+  | PauseResponse
+  | FrameUpdateResponse
+  | LoadStartResponse
+  | RenderingErrorResponse
+  | EffectUpdateResponse;
+
+export type VideoWorkerResponseMessageEvent = MessageEvent<VideoWorkerResponse>;

sam2/demo/frontend/src/common/components/video/editor/DemoVideoEditor.tsx

@@ -0,0 +1,310 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletsAnnotation from '@/common/components/annotations/TrackletsAnnotation';
+import useCloseSessionBeforeUnload from '@/common/components/session/useCloseSessionBeforeUnload';
+import MessagesSnackbar from '@/common/components/snackbar/MessagesSnackbar';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import VideoFilmstripWithPlayback from '@/common/components/video/VideoFilmstripWithPlayback';
+import {
+  FrameUpdateEvent,
+  RenderingErrorEvent,
+  SessionStartedEvent,
+  TrackletsEvent,
+} from '@/common/components/video/VideoWorkerBridge';
+import VideoEditor from '@/common/components/video/editor/VideoEditor';
+import useResetDemoEditor from '@/common/components/video/editor/useResetEditor';
+import useVideo from '@/common/components/video/editor/useVideo';
+import InteractionLayer from '@/common/components/video/layers/InteractionLayer';
+import {PointsLayer} from '@/common/components/video/layers/PointsLayer';
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import UploadLoadingScreen from '@/common/loading/UploadLoadingScreen';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {
+  activeTrackletObjectIdAtom,
+  frameIndexAtom,
+  isAddObjectEnabledAtom,
+  isPlayingAtom,
+  isVideoLoadingAtom,
+  pointsAtom,
+  sessionAtom,
+  streamingStateAtom,
+  trackletObjectsAtom,
+  uploadingStateAtom,
+  VideoData,
+} from '@/demo/atoms';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {color, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useEffect, useState} from 'react';
+import type {ErrorObject} from 'serialize-error';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    overflow: 'auto',
+    width: '100%',
+    borderColor: color['gray-800'],
+    backgroundColor: color['gray-800'],
+    borderWidth: 8,
+    borderRadius: 12,
+    '@media screen and (max-width: 768px)': {
+      // on mobile, we want to grow the editor container so that the editor
+      // fills the remaining vertical space between the navbar and bottom
+      // of the page
+      flexGrow: 1,
+      borderWidth: 0,
+      borderRadius: 0,
+      paddingBottom: spacing[4],
+    },
+  },
+  loadingScreenWrapper: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    width: '100%',
+    height: '100%',
+    background: 'white',
+    overflow: 'hidden',
+    overflowY: 'auto',
+    zIndex: 999,
+  },
+});
+
+type Props = {
+  video: VideoData;
+};
+
+export default function DemoVideoEditor({video: inputVideo}: Props) {
+  const {settings} = useSettingsContext();
+  const video = useVideo();
+
+  const [isSessionStartFailed, setIsSessionStartFailed] =
+    useState<boolean>(false);
+
+  const [session, setSession] = useAtom(sessionAtom);
+
+  const [activeTrackletId, setActiveTrackletObjectId] = useAtom(
+    activeTrackletObjectIdAtom,
+  );
+  const setTrackletObjects = useSetAtom(trackletObjectsAtom);
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const points = useAtomValue(pointsAtom);
+  const isAddObjectEnabled = useAtomValue(isAddObjectEnabledAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isVideoLoading = useAtomValue(isVideoLoadingAtom);
+  const uploadingState = useAtomValue(uploadingStateAtom);
+
+  const [renderingError, setRenderingError] = useState<ErrorObject | null>(
+    null,
+  );
+
+  const {isMobile} = useScreenSize();
+
+  const [tabIndex] = useToolbarTabs();
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useCloseSessionBeforeUnload();
+
+  const {resetEditor, resetSession} = useResetDemoEditor();
+  useEffect(() => {
+    resetEditor();
+  }, [inputVideo, resetEditor]);
+
+  useEffect(() => {
+    function onFrameUpdate(event: FrameUpdateEvent) {
+      setFrameIndex(event.index);
+    }
+
+    // Listen to frame updates to fetch the frame index in the main thread,
+    // which is then used downstream to render points per frame.
+    video?.addEventListener('frameUpdate', onFrameUpdate);
+
+    function onSessionStarted(event: SessionStartedEvent) {
+      setSession({id: event.sessionId, ranPropagation: false});
+    }
+
+    video?.addEventListener('sessionStarted', onSessionStarted);
+
+    function onSessionStartFailed() {
+      setIsSessionStartFailed(true);
+    }
+
+    video?.addEventListener('sessionStartFailed', onSessionStartFailed);
+
+    function onTrackletsUpdated(event: TrackletsEvent) {
+      const tracklets = event.tracklets;
+      if (tracklets.length === 0) {
+        resetSession();
+      }
+      setTrackletObjects(tracklets);
+    }
+
+    video?.addEventListener('trackletsUpdated', onTrackletsUpdated);
+
+    function onRenderingError(event: RenderingErrorEvent) {
+      setRenderingError(event.error);
+    }
+
+    video?.addEventListener('renderingError', onRenderingError);
+
+    video?.initializeTracker('SAM 2', {
+      inferenceEndpoint: settings.inferenceAPIEndpoint,
+    });
+
+    video?.startSession(inputVideo.path);
+
+    return () => {
+      video?.closeSession();
+      video?.removeEventListener('frameUpdate', onFrameUpdate);
+      video?.removeEventListener('sessionStarted', onSessionStarted);
+      video?.removeEventListener('sessionStartFailed', onSessionStartFailed);
+      video?.removeEventListener('trackletsUpdated', onTrackletsUpdated);
+      video?.removeEventListener('renderingError', onRenderingError);
+    };
+  }, [
+    setFrameIndex,
+    setSession,
+    setTrackletObjects,
+    resetSession,
+    inputVideo,
+    video,
+    settings.inferenceAPIEndpoint,
+    settings.videoAPIEndpoint,
+  ]);
+
+  async function handleOptimisticPointUpdate(newPoints: SegmentationPoint[]) {
+    if (session == null) {
+      return;
+    }
+
+    async function createActiveTracklet() {
+      if (!isAddObjectEnabled || newPoints.length === 0) {
+        return;
+      }
+      const tracklet = await video?.createTracklet();
+      if (tracklet != null && newPoints.length > 0) {
+        setActiveTrackletObjectId(tracklet.id);
+        video?.updatePoints(tracklet.id, [newPoints[newPoints.length - 1]]);
+      }
+    }
+
+    if (activeTrackletId != null) {
+      video?.updatePoints(activeTrackletId, newPoints);
+    } else {
+      await createActiveTracklet();
+    }
+    enqueueMessage('pointClick');
+  }
+
+  async function handleAddPoint(point: SegmentationPoint) {
+    if (streamingState === 'partial' || streamingState === 'requesting') {
+      return;
+    }
+    if (isPlaying) {
+      return video?.pause();
+    }
+    handleOptimisticPointUpdate([...points, point]);
+  }
+
+  function handleRemovePoint(point: SegmentationPoint) {
+    if (
+      isPlaying ||
+      streamingState === 'partial' ||
+      streamingState === 'requesting'
+    ) {
+      return;
+    }
+    handleOptimisticPointUpdate(points.filter(p => p !== point));
+  }
+
+  // The interaction layer handles clicks onto the video canvas. It is used
+  // to get absolute point clicks within the video's coordinate system.
+  // The PointsLayer handles rendering of input points and allows removing
+  // individual points by clicking on them.
+  const layers = (
+    <>
+      {tabIndex === OBJECT_TOOLBAR_INDEX && (
+        <>
+          <InteractionLayer
+            key="interaction-layer"
+            onPoint={point => handleAddPoint(point)}
+          />
+          <PointsLayer
+            key="points-layer"
+            points={points}
+            onRemovePoint={handleRemovePoint}
+          />
+        </>
+      )}
+      {!isMobile && <MessagesSnackbar key="snackbar-layer" />}
+    </>
+  );
+
+  return (
+    <>
+      {(isVideoLoading || session === null) && !isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Loading demo..."
+            description="This may take a few moments, you're almost there!"
+          />
+        </div>
+      )}
+      {isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Did we just break the internet?"
+            description={
+              <>Uh oh, it looks like there was an issue starting a session.</>
+            }
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {isMobile && renderingError != null && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Well, this is embarrassing..."
+            description="This demo is not optimized for your device. Please try again on a different device with a larger screen."
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {uploadingState !== 'default' && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <UploadLoadingScreen />
+        </div>
+      )}
+      <div {...stylex.props(styles.container)}>
+        <VideoEditor
+          video={inputVideo}
+          layers={layers}
+          loading={session == null}>
+          <div className="bg-graydark-800 w-full">
+            <VideoFilmstripWithPlayback />
+            <TrackletsAnnotation />
+          </div>
+        </VideoEditor>
+      </div>
+    </>
+  );
+}

sam2/demo/frontend/src/common/components/video/editor/ImageUtils.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function convertVideoFrameToImageData(
+  videoFrame: VideoFrame,
+): ImageData | undefined {
+  const canvas = new OffscreenCanvas(
+    videoFrame.displayWidth,
+    videoFrame.displayHeight,
+  );
+  const ctx = canvas.getContext('2d');
+  ctx?.drawImage(videoFrame, 0, 0);
+  return ctx?.getImageData(0, 0, canvas.width, canvas.height);
+}
+
+/**
+ * This utility provides two functions:
+ * `process`: to find the bounding box of non-empty pixels from an ImageData, when looping through all its pixels
+ * `crop` to cut out the subsection found in `process`
+ * @returns
+ */
+export function findBoundingBox() {
+  let xMin = Number.MAX_VALUE;
+  let yMin = Number.MAX_VALUE;
+  let xMax = Number.MIN_VALUE;
+  let yMax = Number.MIN_VALUE;
+  return {
+    process: function (x: number, y: number, hasData: boolean) {
+      if (hasData) {
+        xMin = Math.min(x, xMin);
+        xMax = Math.max(x, xMax);
+        yMin = Math.min(y, yMin);
+        yMax = Math.max(y, yMax);
+      }
+      return [xMin, xMax, yMin, yMax];
+    },
+    crop(imageData: ImageData): ImageData | null {
+      const canvas = new OffscreenCanvas(imageData.width, imageData.height);
+      const ctx = canvas.getContext('2d');
+
+      const boundingBoxWidth = xMax - xMin;
+      const boundingBoxHeight = yMax - yMin;
+      if (ctx && boundingBoxWidth > 0 && boundingBoxHeight > 0) {
+        ctx.clearRect(0, 0, canvas.width, canvas.height);
+        ctx.putImageData(imageData, 0, 0);
+        return ctx.getImageData(
+          xMin,
+          yMin,
+          boundingBoxWidth,
+          boundingBoxHeight,
+        );
+      } else {
+        return null;
+      }
+    },
+    getBox(): [[number, number], [number, number]] {
+      return [
+        [xMin, yMin],
+        [xMax, yMax],
+      ];
+    },
+  };
+}
+
+export function magnifyImageRegion(
+  canvas: HTMLCanvasElement | null,
+  x: number,
+  y: number,
+  radius: number = 25,
+  scale: number = 2,
+): string {
+  if (canvas == null) {
+    return '';
+  }
+  const ctx = canvas.getContext('2d');
+  if (ctx) {
+    const minX = x - radius < 0 ? radius - x : 0;
+    const minY = y - radius < 0 ? radius - y : 0;
+    const region = ctx.getImageData(
+      Math.max(x - radius, 0),
+      Math.max(y - radius, 0),
+      radius * 2,
+      radius * 2,
+    );
+
+    // ImageData doesn't scale-transform correctly on canvas
+    // So we first draw the original size on an offscreen canvas, and then scale it
+    const regionCanvas = new OffscreenCanvas(region.width, region.height);
+    const regionCtx = regionCanvas.getContext('2d');
+    regionCtx?.putImageData(region, minX > 0 ? minX : 0, minY > 0 ? minY : 0);
+
+    const scaleCanvas = document.createElement('canvas');
+    scaleCanvas.width = Math.round(region.width * scale);
+    scaleCanvas.height = Math.round(region.height * scale);
+    const scaleCtx = scaleCanvas.getContext('2d');
+    scaleCtx?.scale(scale, scale);
+    scaleCtx?.drawImage(regionCanvas, 0, 0);
+
+    return scaleCanvas.toDataURL();
+  }
+  return '';
+}

sam2/demo/frontend/src/common/components/video/editor/VideoEditor.tsx

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {VideoData} from '@/demo/atoms';
+import stylex, {StyleXStyles} from '@stylexjs/stylex';
+import {useSetAtom} from 'jotai';
+import {PropsWithChildren, RefObject, useEffect, useRef} from 'react';
+import Video, {VideoRef} from '../Video';
+import {videoAtom} from './atoms';
+
+const MAX_VIDEO_WIDTH = 1280;
+
+const styles = stylex.create({
+  editorContainer: {
+    position: 'relative',
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    width: '100%',
+    height: '100%',
+    borderRadius: '0.375rem',
+    overflow: {
+      default: 'clip',
+      '@media screen and (max-width: 768px)': 'visible',
+    },
+  },
+  videoContainer: {
+    position: 'relative',
+    flexGrow: 1,
+    overflow: 'hidden',
+    width: '100%',
+    maxWidth: MAX_VIDEO_WIDTH,
+  },
+  layers: {
+    position: 'absolute',
+    left: 0,
+    top: 0,
+    bottom: 0,
+    right: 0,
+  },
+  loadingMessage: {
+    position: 'absolute',
+    top: '8px',
+    right: '8px',
+    padding: '6px 10px',
+    backgroundColor: '#6441D2CC',
+    color: '#FFF',
+    display: 'flex',
+    alignItems: 'center',
+    gap: '8px',
+    borderRadius: '8px',
+    fontSize: '0.8rem',
+  },
+});
+
+export type InteractionLayerProps = {
+  style: StyleXStyles;
+  videoRef: RefObject<VideoRef>;
+};
+
+export type ControlsProps = {
+  isPlaying: boolean;
+  onPlay: () => void;
+  onPause: () => void;
+  onPreviousFrame?: () => void;
+  onNextFrame?: () => void;
+};
+
+type Props = PropsWithChildren<{
+  video: VideoData;
+  layers?: React.ReactNode;
+  loading?: boolean;
+}>;
+
+export default function VideoEditor({
+  video: inputVideo,
+  layers,
+  loading,
+  children,
+}: Props) {
+  const videoRef = useRef<VideoRef>(null);
+  const setVideo = useSetAtom(videoAtom);
+
+  // Initialize video atom
+  useEffect(() => {
+    setVideo(videoRef.current);
+    return () => {
+      setVideo(null);
+    };
+  }, [setVideo]);
+
+  return (
+    <div {...stylex.props(styles.editorContainer)}>
+      <div {...stylex.props(styles.videoContainer)}>
+        <Video
+          ref={videoRef}
+          src={inputVideo.url}
+          width={inputVideo.width}
+          height={inputVideo.height}
+          loading={loading}
+        />
+        <div {...stylex.props(styles.layers)}>{layers}</div>
+      </div>
+      {children}
+    </div>
+  );
+}