need 1 billion dollars for this project

package-lock.json

@@ -54,6 +54,7 @@
         "alchemy-sdk": "^3.4.1",
         "autoprefixer": "10.4.20",
         "axios": "^1.7.4",
+        "bellhop-iframe": "^3.5.0",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.1",
         "cmdk": "^1.0.0",
@@ -3537,6 +3538,11 @@
       "resolved": "https://registry.npmjs.org/bech32/-/bech32-1.1.4.tgz",
       "integrity": "sha512-s0IrSOzLlbvX7yp4WBfPITzpAU8sqQcpsmwXDiKwrG4r491vwCO/XpejasRNl0piBMe/DvP4Tz0mIS/X1DPJBQ=="
     },
+    "node_modules/bellhop-iframe": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/bellhop-iframe/-/bellhop-iframe-3.5.0.tgz",
+      "integrity": "sha512-3KrYh7ZMdNiHvngKyR+3swD0jAiIEOb3clWmmDXMm8KCCNrIy7rGrFUC8mJX/f5X3y8Rwz8lsZeW0i2uUFeVnA=="
+    },
     "node_modules/binary-extensions": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",

package.json

@@ -56,6 +56,7 @@
     "alchemy-sdk": "^3.4.1",
     "autoprefixer": "10.4.20",
     "axios": "^1.7.4",
+    "bellhop-iframe": "^3.5.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
@@ -82,6 +83,7 @@
     "qs": "^6.13.0",
     "query-string": "^9.1.0",
     "react": "18.2.0",
+    "react-avatar": "^5.0.3",
     "react-circular-progressbar": "^2.1.0",
     "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "18.2.0",
@@ -89,7 +91,6 @@
     "react-icons": "^4.12.0",
     "react-photo-sphere-viewer": "^5.0.2-psv5.7.1",
     "react-smooth-scroll-hook": "^1.3.4",
-    "react-avatar": "^5.0.3",
     "react-tuby": "^0.1.24",
     "react-virtualized-auto-sizer": "^1.0.20",
     "react-window-infinite-loader": "^1.0.9",

src/app/page.tsx

@@ -104,7 +104,7 @@ export default async function Page({ searchParams: { v: videoId } }: AppQueryPro
    
       ">
       <h1 className="text-yellow-400/90 text-6xl font-thin">Say goodbye to static videos.</h1>
-      <p className="mt-6 text-white/80 text-xl font-thin">Beta planned for Winter 2024. Follow <a href="x.com/@flngr" className="font-normal font-mono text-stone-50/60 hover:text-stone-50/80 hover:underline hover:underline-offset-2" target="_blank">@flngr</a> for updates.</p>
+      <p className="mt-6 text-white/80 text-xl font-thin">Coming in 2025. Follow <a href="https://x.com/@flngr" className="font-normal font-mono text-stone-50/60 hover:text-stone-50/80 hover:underline hover:underline-offset-2" target="_blank">@flngr</a> for updates.</p>
       </div>
     </div>
   )

src/components/interface/latent-engine/core/engine-clapper.tsx

@@ -0,0 +1,53 @@
+"use client"
+
+import React, { useEffect } from "react"
+
+import { cn } from "@/lib/utils/cn"
+import { MediaInfo } from "@/types/general"
+import { theSimps } from "@/app/latent/samples"
+import { useChildController } from "./useChildController"
+import { useSetupIframeOnce } from "./useSetupIframeOnce"
+
+function LatentEngineClapper({
+  media,
+  width,
+  height,
+  className = "" }: {
+  media: MediaInfo
+  width?: number
+  height?: number
+  className?: string
+}) {
+  // only call this once per iframe
+  useSetupIframeOnce()
+
+  const hasLoadedBellhop = useChildController(s => s.hasLoadedBellhop)
+  const sendMessage = useChildController(s => s.sendMessage)
+
+  useEffect(() => {
+    console.log('connected to the iframe player, now loading the prompt..')
+    sendMessage('loadPrompt', { prompt: theSimps })
+  }, [media.id, hasLoadedBellhop])
+
+  return (
+    <div
+      style={{ width, height }}
+      className={cn(`
+      relative
+      flex flex-col
+      items-center justify-between
+      w-full h-full
+      rounded-xl overflow-hidden
+      bg-black
+      `, className)}>
+      <iframe
+        className="pointer-events-auto"
+        width={width}
+        height={height}
+        src={`http://localhost:3000/embed?clap=/samples/claps/wasteland.clap`}
+      />
+    </div>
+  );
+}
+
+export default LatentEngineClapper

src/components/interface/latent-engine/core/{engine.tsx → engine-legacy.tsx}

@@ -17,7 +17,7 @@ import { useStore } from "@/app/state/useStore"
 import { theSimps } from "@/app/latent/samples"
 import { generateClapFromPrompt } from "./generateClapFromPrompt"
 
-function LatentEngine({
+function LatentEngineLegacy({
   media,
   width,
   height,
@@ -291,4 +291,4 @@ function LatentEngine({
   );
 }
 
-export default LatentEngine
+export default LatentEngineLegacy

src/components/interface/latent-engine/core/generateClapFromPrompt.ts

@@ -85,7 +85,7 @@ export function generateClapFromPrompt({
  
 
   for (let prompt of story) {
-
+    /*
     clap.segments.push(newSegment({
       track: 0,
       startTimeInMs,
@@ -94,6 +94,16 @@ export function generateClapFromPrompt({
       prompt: "",
       label: "video",
       outputType: ClapOutputType.VIDEO,
+    }))
+      */
+    clap.segments.push(newSegment({
+      track: 0,
+      startTimeInMs,
+      endTimeInMs,
+      category: ClapSegmentCategory.STORYBOARD,
+      prompt: "",
+      label: "movie screencap",
+      outputType: ClapOutputType.IMAGE,
     }))
     clap.segments.push(newSegment({
       track: 1,

src/components/interface/latent-engine/core/useChildController.ts

@@ -0,0 +1,74 @@
+'use client'
+
+import { create } from 'zustand'
+import { Bellhop } from 'bellhop-iframe'
+
+export const useChildController = create<{
+  bellhop: Bellhop
+
+  /**
+   * Whether the communication pipeline seems to be working or not
+   *
+   * Initially we assume that it is, but this can be invalidated
+   * if we know for certain it is not, or in case of exception
+   */
+  canUseBellhop: boolean
+
+  /**
+   * 
+   */
+  hasLoadedBellhop: boolean
+
+  /**
+   * Whether Clapper is ready
+   * 
+   * This will be set upon reception of the event
+   */
+  clapperIsReady: boolean
+
+  setDomElement: (domElement: HTMLIFrameElement, origin?: string) => void
+
+  setCanUseBellhop: (canUseBellhop: boolean) => void
+  setHasLoadedBellhop: (hasLoadedBellhop: boolean) => void
+  onMessage: (name: string, callback: Function, priority?: number) => void
+  sendMessage: (type: string, data?: any) => void
+}>((set, get) => ({
+  bellhop: undefined as unknown as Bellhop,
+  canUseBellhop: true,
+  hasLoadedBellhop: false,
+  clapperIsReady: false,
+  setDomElement: (domElement: HTMLIFrameElement, origin?: string) => {
+    const bellhop = new Bellhop()
+    bellhop.connect(domElement, origin)
+    set({
+      bellhop,
+    })
+  },
+  setCanUseBellhop: (canUseBellhop: boolean) => {
+    set({
+      canUseBellhop,
+    })
+  },
+  setHasLoadedBellhop: (hasLoadedBellhop: boolean) => {
+    set({
+      bellhop: hasLoadedBellhop ? new Bellhop() : (undefined as unknown as Bellhop),
+      hasLoadedBellhop,
+    })
+  },
+  onMessage: (name: string, callback: Function, priority?: number) => {
+    const { bellhop } = get()
+    try {
+      bellhop.on(name, callback, priority)
+    } catch (err) {
+      console.log(`failed to subscribe to parent iframe messages:`, err)
+    }
+  },
+  sendMessage: (type: string, data?: any) => {
+    const { bellhop } = get()
+    try {
+      bellhop.send(type, data)
+    } catch (err) {
+      console.log(`failed to send a message to parent iframe:`, err)
+    }
+  },
+}))

src/components/interface/latent-engine/core/useSetupIframeOnce.ts

@@ -0,0 +1,65 @@
+'use client'
+
+import { useEffect, useRef } from 'react'
+import { useChildController } from './useChildController'
+
+/**
+ * You should only call this once, at the root of the react tree
+ */
+export function useSetupIframeOnce() {
+  const ref = useRef<HTMLIFrameElement>(null)
+  const setDomElement = useChildController(s => s.setDomElement)
+  const canUseBellhop = useChildController((s) => s.canUseBellhop)
+  const setCanUseBellhop = useChildController((s) => s.setCanUseBellhop)
+  const isConnectedToChild = useChildController((s) => s.isConnectedToChild)
+  const setHasLoadedBellhop = useChildController((s) => s.setHasLoadedBellhop)
+  const onMessage = useChildController((s) => s.onMessage)
+  const sendMessage = useChildController((s) => s.sendMessage)
+
+
+  // TODO: maybe we should add a JWT token to secure this, make it only embeddable
+  // on a certain website (eg. AiTube.at), and if people want to
+  // embed the player somewhere's else they will have to deploy their own
+
+  const domElement = ref.current
+
+  useEffect(() => {
+    if (!domElement || !isConnectedToChild) {
+      // when we are detecting that we are not in an iframe
+
+      if (canUseBellhop) {
+        setCanUseBellhop(false)
+      }
+      return
+    }
+
+    if (!canUseBellhop) {
+      setCanUseBellhop(true)
+    }
+
+    if (isConnectedToChild) {
+      // no need to connect twice
+      return
+    } else {
+      // we only try this once
+      
+
+      try {
+        setDomElement(domElement)
+        setHasLoadedBellhop(true)
+
+        onMessage('something', function (event: any) {
+          // generate the first scene of an OpenClap file from the prompt
+        })
+
+        sendMessage('status', { isReady: true })
+      } catch (err) {
+        console.error(`failed to initialize bellhop`)
+        setHasLoadedBellhop(false)
+        setCanUseBellhop(false)
+      }
+    }
+  }, [domElement, canUseBellhop, isConnectedToChild])
+
+  return useChildController
+}

src/components/interface/latent-engine/index.tsx

@@ -2,6 +2,6 @@
 
 import dynamic from "next/dynamic"
 
-export const LatentEngine = dynamic(() => import("./core/engine"), {
+export const LatentEngine = dynamic(() => import("./core/engine-clapper"), {
   loading: () => null,
 })