+
+ Object {tracklet.id + 1}
+
+
+
+ {segments.map(segment => {
+ return (
+
+ );
+ })}
+ {framesWithPoints.map(index => {
+ return (
+
{
+ onSelectFrame?.(tracklet, index);
+ }}
+ {...stylex.props(styles.segmentationPoint)}
+ style={{
+ left: Math.floor(selection.toPosition(index) - 4),
+ backgroundColor: tracklet.color,
+ }}
+ />
+ );
+ })}
+
+
+ );
+}
diff --git a/demo/frontend/src/common/components/annotations/TrackletsAnnotation.tsx b/demo/frontend/src/common/components/annotations/TrackletsAnnotation.tsx
new file mode 100644
index 0000000..aa16223
--- /dev/null
+++ b/demo/frontend/src/common/components/annotations/TrackletsAnnotation.tsx
@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletSwimlane from '@/common/components/annotations/TrackletSwimlane';
+import useTracklets from '@/common/components/annotations/useTracklets';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import {m, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+ container: {
+ marginTop: m[3],
+ height: 75,
+ paddingHorizontal: spacing[4],
+ '@media screen and (max-width: 768px)': {
+ height: 25,
+ },
+ },
+});
+
+export default function TrackletsAnnotation() {
+ const video = useVideo();
+ const tracklets = useTracklets();
+
+ function handleSelectFrame(_tracklet: BaseTracklet, index: number) {
+ if (video !== null) {
+ video.frame = index;
+ }
+ }
+
+ return (
+
+ {tracklets.map(tracklet => (
+
+ ))}
+
+ );
+}
diff --git a/demo/frontend/src/common/components/annotations/useTracklets.ts b/demo/frontend/src/common/components/annotations/useTracklets.ts
new file mode 100644
index 0000000..b709572
--- /dev/null
+++ b/demo/frontend/src/common/components/annotations/useTracklets.ts
@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {trackletObjectsAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function useTracklets() {
+ return useAtomValue(trackletObjectsAtom);
+}
diff --git a/demo/frontend/src/common/components/button/GradientBorder.tsx b/demo/frontend/src/common/components/button/GradientBorder.tsx
new file mode 100644
index 0000000..3be6e65
--- /dev/null
+++ b/demo/frontend/src/common/components/button/GradientBorder.tsx
@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import stylex from '@stylexjs/stylex';
+
+import {gradients} from '@/theme/tokens.stylex';
+
+enum GradientTypes {
+ fullGradient = 'fullGradient',
+ bluePinkGradient = 'bluePinkGradient',
+}
+
+type Props = {
+ gradientType?: GradientTypes;
+ disabled?: boolean;
+ rounded?: boolean;
+ className?: string;
+} & React.DOMAttributes
;
+
+const styles = stylex.create({
+ animationHover: {
+ ':hover': {
+ backgroundPosition: '300% 100%',
+ },
+ },
+
+ fullGradient: {
+ border: '2px solid transparent',
+ background: gradients['rainbow'],
+ backgroundSize: '100% 400%',
+ transition: 'background 0.35s ease-in-out',
+ },
+
+ bluePinkGradient: {
+ border: '2px solid transparent',
+ background: gradients['rainbow'],
+ },
+});
+
+export default function GradientBorder({
+ gradientType = GradientTypes.fullGradient,
+ disabled,
+ rounded = true,
+ className = '',
+ children,
+}: Props) {
+ const gradient = (name: GradientTypes) => {
+ if (name === GradientTypes.fullGradient) {
+ return styles.fullGradient;
+ } else if (name === GradientTypes.bluePinkGradient) {
+ return styles.bluePinkGradient;
+ }
+ };
+
+ return (
+
+ {children}
+
+ );
+}
diff --git a/demo/frontend/src/common/components/button/PlaybackButton.tsx b/demo/frontend/src/common/components/button/PlaybackButton.tsx
new file mode 100644
index 0000000..7da7925
--- /dev/null
+++ b/demo/frontend/src/common/components/button/PlaybackButton.tsx
@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import Tooltip from '@/common/components/Tooltip';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {isPlayingAtom, streamingStateAtom, toolbarTabIndex} from '@/demo/atoms';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import {useAtomValue} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function PlaybackButton() {
+ const tabIndex = useAtomValue(toolbarTabIndex);
+ const streamingState = useAtomValue(streamingStateAtom);
+ const isPlaying = useAtomValue(isPlayingAtom);
+ const video = useVideo();
+
+ const isDisabled =
+ tabIndex === OBJECT_TOOLBAR_INDEX &&
+ streamingState !== 'none' &&
+ streamingState !== 'full';
+
+ const handlePlay = useCallback(() => {
+ video?.play();
+ }, [video]);
+
+ const handlePause = useCallback(() => {
+ video?.pause();
+ }, [video]);
+
+ const handleClick = useCallback(() => {
+ if (isDisabled) {
+ return;
+ }
+ if (isPlaying) {
+ handlePause();
+ } else {
+ handlePlay();
+ }
+ }, [isDisabled, isPlaying, handlePlay, handlePause]);
+
+ useEffect(() => {
+ const handleKey = (event: KeyboardEvent) => {
+ const callback = {
+ KeyK: handleClick,
+ }[event.code];
+ if (callback != null) {
+ event.preventDefault();
+ callback();
+ }
+ };
+ document.addEventListener('keydown', handleKey);
+ return () => {
+ document.removeEventListener('keydown', handleKey);
+ };
+ }, [handleClick]);
+
+ return (
+
+
+
+ );
+}
+
+function getButtonStyles(isDisabled: boolean): string {
+ if (isDisabled) {
+ return '!bg-gray-600 !text-graydark-700';
+ }
+ return `!text-black bg-white`;
+}
diff --git a/demo/frontend/src/common/components/button/PrimaryCTAButton.tsx b/demo/frontend/src/common/components/button/PrimaryCTAButton.tsx
new file mode 100644
index 0000000..3e8f7d6
--- /dev/null
+++ b/demo/frontend/src/common/components/button/PrimaryCTAButton.tsx
@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+ disabled?: boolean;
+ endIcon?: ReactNode;
+} & React.DOMAttributes;
+
+export default function PrimaryCTAButton({
+ children,
+ disabled,
+ endIcon,
+ ...props
+}: Props) {
+ return (
+
+
+
+ );
+}
diff --git a/demo/frontend/src/common/components/button/ResponsiveButton.tsx b/demo/frontend/src/common/components/button/ResponsiveButton.tsx
new file mode 100644
index 0000000..e8aaa6d
--- /dev/null
+++ b/demo/frontend/src/common/components/button/ResponsiveButton.tsx
@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useScreenSize from '@/common/screen/useScreenSize';
+import type {ReactNode} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = ButtonProps & {startIcon: ReactNode};
+
+export default function ResponsiveButton(props: Props) {
+ const {isMobile} = useScreenSize();
+
+ return ;
+}
diff --git a/demo/frontend/src/common/components/button/TrackAndPlayButton.tsx b/demo/frontend/src/common/components/button/TrackAndPlayButton.tsx
new file mode 100644
index 0000000..51213ad
--- /dev/null
+++ b/demo/frontend/src/common/components/button/TrackAndPlayButton.tsx
@@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useFunctionThrottle from '@/common/components/useFunctionThrottle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+ areTrackletObjectsInitializedAtom,
+ isStreamingAtom,
+ sessionAtom,
+ streamingStateAtom,
+} from '@/demo/atoms';
+import {ChevronRight} from '@carbon/icons-react';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function TrackAndPlayButton() {
+ const video = useVideo();
+ const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+ const streamingState = useAtomValue(streamingStateAtom);
+ const areObjectsInitialized = useAtomValue(areTrackletObjectsInitializedAtom);
+ const setSession = useSetAtom(sessionAtom);
+ const {enqueueMessage} = useMessagesSnackbar();
+ const {isThrottled, maxThrottles, throttle} = useFunctionThrottle(250, 4);
+
+ const isTrackAndPlayDisabled =
+ streamingState === 'aborting' || streamingState === 'requesting';
+
+ useEffect(() => {
+ function onStreamingStarted() {
+ setIsStreaming(true);
+ }
+ video?.addEventListener('streamingStarted', onStreamingStarted);
+
+ function onStreamingCompleted() {
+ enqueueMessage('trackAndPlayComplete');
+ setIsStreaming(false);
+ }
+ video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+ return () => {
+ video?.removeEventListener('streamingStarted', onStreamingStarted);
+ video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+ };
+ }, [video, setIsStreaming, enqueueMessage]);
+
+ const handleTrackAndPlay = useCallback(() => {
+ if (isTrackAndPlayDisabled) {
+ return;
+ }
+ if (maxThrottles && isThrottled) {
+ enqueueMessage('trackAndPlayThrottlingWarning');
+ }
+
+ // Throttling is only applied while streaming because we should
+ // only throttle after a user has aborted inference. This way,
+ // a user can still quickly abort a stream if they notice the
+ // inferred mask is misaligned.
+ throttle(
+ () => {
+ if (!isStreaming) {
+ enqueueMessage('trackAndPlayClick');
+ video?.streamMasks();
+ setSession(previousSession =>
+ previousSession == null
+ ? previousSession
+ : {...previousSession, ranPropagation: true},
+ );
+ } else {
+ video?.abortStreamMasks();
+ }
+ },
+ {enableThrottling: isStreaming},
+ );
+ }, [
+ isTrackAndPlayDisabled,
+ isThrottled,
+ isStreaming,
+ maxThrottles,
+ video,
+ setSession,
+ enqueueMessage,
+ throttle,
+ ]);
+
+ useEffect(() => {
+ const handleKey = (event: KeyboardEvent) => {
+ const callback = {
+ KeyK: handleTrackAndPlay,
+ }[event.code];
+ if (callback != null) {
+ event.preventDefault();
+ callback();
+ }
+ };
+ document.addEventListener('keydown', handleKey);
+ return () => {
+ document.removeEventListener('keydown', handleKey);
+ };
+ }, [handleTrackAndPlay]);
+
+ return (
+ }>
+ {isStreaming ? 'Cancel Tracking' : 'Track objects'}
+
+ );
+}
diff --git a/demo/frontend/src/common/components/code/InitializeLocalMonaco.ts b/demo/frontend/src/common/components/code/InitializeLocalMonaco.ts
new file mode 100644
index 0000000..3d332ac
--- /dev/null
+++ b/demo/frontend/src/common/components/code/InitializeLocalMonaco.ts
@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {loader} from '@monaco-editor/react';
+
+import Logger from '@/common/logger/Logger';
+import * as monaco from 'monaco-editor';
+import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker';
+import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker';
+
+self.MonacoEnvironment = {
+ getWorker(_, label) {
+ if (label === 'typescript' || label === 'javascript') {
+ return new tsWorker();
+ }
+ return new editorWorker();
+ },
+};
+
+loader.config({monaco});
+
+loader.init().then(monaco => {
+ Logger.debug('initialized monaco', monaco);
+});
diff --git a/demo/frontend/src/common/components/effects/BackgroundEffects.tsx b/demo/frontend/src/common/components/effects/BackgroundEffects.tsx
new file mode 100644
index 0000000..84aa6bb
--- /dev/null
+++ b/demo/frontend/src/common/components/effects/BackgroundEffects.tsx
@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {backgroundEffects} from '@/common/components/effects/EffectsUtils';
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {activeBackgroundEffectAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function BackgroundEffects() {
+ const setEffect = useVideoEffect();
+ const activeEffect = useAtomValue(activeBackgroundEffectAtom);
+
+ return (
+
+ {backgroundEffects.map(backgroundEffect => {
+ return (
+
+ )
+ }
+ onClick={() => {
+ if (activeEffect.name === backgroundEffect.effectName) {
+ setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND, {
+ variant:
+ (activeEffect.variant + 1) % activeEffect.numVariants,
+ });
+ } else {
+ setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND);
+ }
+ }}
+ />
+ );
+ })}
+
+ );
+}
diff --git a/demo/frontend/src/common/components/effects/EffectVariantBadge.tsx b/demo/frontend/src/common/components/effects/EffectVariantBadge.tsx
new file mode 100644
index 0000000..99df4b5
--- /dev/null
+++ b/demo/frontend/src/common/components/effects/EffectVariantBadge.tsx
@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {right, top} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+ variantBadge: {
+ position: 'absolute',
+ top: top[1],
+ right: right[1],
+ backgroundColor: '#280578',
+ color: '#D2D2FF',
+ fontVariantNumeric: 'tabular-nums',
+ paddingHorizontal: 4,
+ paddingVertical: 1,
+ fontSize: 9,
+ borderRadius: 6,
+ fontWeight: 'bold',
+ },
+});
+
+type Props = {
+ label: string;
+};
+
+export default function VariantBadge({label}: Props) {
+ return {label}
;
+}
diff --git a/demo/frontend/src/common/components/effects/EffectsCarousel.tsx b/demo/frontend/src/common/components/effects/EffectsCarousel.tsx
new file mode 100644
index 0000000..6a05c62
--- /dev/null
+++ b/demo/frontend/src/common/components/effects/EffectsCarousel.tsx
@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {CarouselContainerShadow} from '@/common/components/effects/EffectsCarouselShadow';
+import {DemoEffect} from '@/common/components/effects/EffectsUtils';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import type {EffectIndex} from '@/common/components/video/effects/Effects';
+import {Effects} from '@/common/components/video/effects/Effects';
+import {color, fontSize, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+type Props = {
+ label: string;
+ effects: DemoEffect[];
+ activeEffect: keyof Effects;
+ index: EffectIndex;
+};
+
+const styles = stylex.create({
+ container: {
+ display: 'flex',
+ flexDirection: 'column',
+ gap: spacing[2],
+ width: '100%',
+ },
+ label: {
+ fontSize: fontSize['xs'],
+ color: '#A6ACB2',
+ textAlign: 'center',
+ },
+ carouselContainer: {
+ position: 'relative',
+ borderRadius: '8px',
+ overflow: 'hidden',
+ width: '100%',
+ height: '120px',
+ backgroundColor: color['gray-700'],
+ },
+});
+
+export default function EffectsCarousel({
+ label,
+ effects,
+ activeEffect,
+ index: effectIndex,
+}: Props) {
+ const setEffect = useVideoEffect();
+
+ return (
+
+
{label}
+
+
+
+
+ {effects.map(({effectName, Icon, title}, index) => {
+ const isActive = activeEffect === effectName;
+ return (
+
setEffect(effectName, effectIndex)}>
+
+
+ {title}
+
+
+ );
+ })}
+
+
+
+
+