Improve portal editor and rebake all default scene portals (#9588)

* Improve portal editor and rebake all default scene portals

* cleanup logs

Author: HexaField

packages/client-core/i18n/en/editor.json

@@ -806,8 +806,8 @@
       "lbl-effectType": "Loading Effect",
       "lbl-previewType": "Preview Type",
       "lbl-previewImageURL": "Preview Image URL",
-      "lbl-savedImageURL": "Saved Image URL",
-      "lbl-previewImage": "Preview Image",
+      "lbl-savedImageURL": "Saved Image",
+      "lbl-generateImage": "Generate Image",
       "lbl-saveImage": "Save Image",
       "lbl-spawnPosition": "Spawn Position",
       "lbl-spawnRotation": "Spawn Rotation",

packages/editor/src/components/inputs/FileBrowserInput.tsx

@@ -58,7 +58,7 @@ export function FileBrowserInput({
   const onUpload = useUpload(uploadOptions)
 
   // todo fix for invalid URLs
-  const assetIsExternal = value && !value?.includes(config.client.fileServer)
+  const assetIsExternal = value && !value?.includes(config.client.fileServer) && !value.includes('blob:https://')
   const uploadExternalAsset = () => {
     onUpload([
       {

packages/editor/src/components/properties/PortalNodeEditor.tsx

@@ -23,10 +23,9 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
 Ethereal Engine. All Rights Reserved.
 */
 
-import { debounce } from 'lodash'
-import React, { useCallback, useEffect, useState } from 'react'
+import React, { useEffect } from 'react'
 import { useTranslation } from 'react-i18next'
-import { Euler, Quaternion } from 'three'
+import { Euler, Quaternion, Vector3 } from 'three'
 
 import { API } from '@etherealengine/client-core/src/API'
 import { getComponent, useComponent } from '@etherealengine/engine/src/ecs/functions/ComponentFunctions'
@@ -42,7 +41,9 @@ import { TransformComponent } from '@etherealengine/engine/src/transform/compone
 import MeetingRoomIcon from '@mui/icons-material/MeetingRoom'
 
 import { PortalType, portalPath } from '@etherealengine/common/src/schema.type.module'
-import { getPreviewBakeTexture, uploadCubemapBakeToServer } from '../../functions/uploadEnvMapBake'
+import { imageDataToBlob } from '@etherealengine/engine/src/scene/classes/ImageUtils'
+import { useHookstate } from '@hookstate/core'
+import { bakeEnvmapTexture, uploadCubemapBakeToServer } from '../../functions/uploadEnvMapBake'
 import BooleanInput from '../inputs/BooleanInput'
 import { Button } from '../inputs/Button'
 import EulerInput from '../inputs/EulerInput'
@@ -52,17 +53,11 @@ import SelectInput from '../inputs/SelectInput'
 import { ControlledStringInput } from '../inputs/StringInput'
 import Vector3Input from '../inputs/Vector3Input'
 import NodeEditor from './NodeEditor'
-import { EditorComponentType, commitProperties, commitProperty, updateProperties, updateProperty } from './Util'
+import { EditorComponentType, commitProperties, commitProperty, updateProperty } from './Util'
 
 type PortalOptions = {
-  name: string
-  value: string
-}
-
-type PortalFilterOption = {
   label: string
   value: string
-  data: PortalOptions
 }
 
 const rotation = new Quaternion()
@@ -73,8 +68,11 @@ const rotation = new Quaternion()
  * @type {class component}
  */
 export const PortalNodeEditor: EditorComponentType = (props) => {
-  const [portals, setPortals] = useState<Array<{ value: string; label: string }>>([])
-  const [bufferUrl, setBufferUrl] = useState<string>('')
+  const state = useHookstate({
+    portals: [] as PortalOptions[],
+    previewImageData: null as ImageData | null,
+    previewImageURL: ''
+  })
 
   const { t } = useTranslation()
   const transformComponent = useComponent(props.entity, TransformComponent)
@@ -85,20 +83,14 @@ export const PortalNodeEditor: EditorComponentType = (props) => {
   }, [])
 
   const updateCubeMapBake = async () => {
-    const imageBlob = await getPreviewBakeTexture(transformComponent.value.position)
-    const url = URL.createObjectURL(imageBlob)
-    setBufferUrl(url)
+    const imageData = await bakeEnvmapTexture(
+      transformComponent.value.position.clone().add(new Vector3(0, 2, 0).multiply(transformComponent.scale.value))
+    )
+    const blob = await imageDataToBlob(imageData)
+    state.previewImageData.set(imageData)
+    state.previewImageURL.set(URL.createObjectURL(blob!))
   }
 
-  const updateCubeMapBakeDebounced = useCallback(debounce(updateCubeMapBake, 500), []) //ms
-
-  useEffect(() => {
-    updateCubeMapBakeDebounced()
-    return () => {
-      updateCubeMapBakeDebounced.cancel()
-    }
-  }, [transformComponent.position])
-
   const loadPortals = async () => {
     const portalsDetail: PortalType[] = []
     try {
@@ -109,7 +101,7 @@ export const PortalNodeEditor: EditorComponentType = (props) => {
     } catch (error) {
       throw new Error(error)
     }
-    setPortals(
+    state.portals.set(
       portalsDetail
         .filter((portal) => portal.portalEntityId !== getComponent(props.entity, UUIDComponent))
         .map(({ portalEntityId, portalEntityName, sceneName }) => {
@@ -118,19 +110,16 @@ export const PortalNodeEditor: EditorComponentType = (props) => {
     )
   }
 
-  const bakeCubemap = async () => {
-    const url = await uploadCubemapBakeToServer(
-      getComponent(props.entity, NameComponent),
-      transformComponent.position.value
-    )
-    loadPortals()
-    updateProperties(PortalComponent, { previewImageURL: url }, [props.entity])
+  const uploadEnvmap = async () => {
+    if (!state.previewImageData.value) return
+    const url = await uploadCubemapBakeToServer(getComponent(props.entity, NameComponent), state.previewImageData.value)
+    commitProperties(PortalComponent, { previewImageURL: url }, [props.entity])
   }
 
   const changeSpawnRotation = (value: Euler) => {
     rotation.setFromEuler(value)
 
-    updateProperties(PortalComponent, { spawnRotation: rotation })
+    commitProperties(PortalComponent, { spawnRotation: rotation })
   }
 
   const changePreviewType = (val) => {
@@ -150,7 +139,7 @@ export const PortalNodeEditor: EditorComponentType = (props) => {
       <InputGroup name="Portal" label={t('editor:properties.portal.lbl-portal')}>
         <SelectInput
           key={props.entity}
-          options={portals}
+          options={state.portals.value}
           value={portalComponent.linkedPortalId.value}
           onChange={commitProperty(PortalComponent, 'linkedPortalId')}
         />
@@ -185,14 +174,14 @@ export const PortalNodeEditor: EditorComponentType = (props) => {
           onRelease={commitProperty(PortalComponent, 'previewImageURL')}
         />
       </InputGroup>
-      <InputGroup name="Preview Image Bake" label={t('editor:properties.portal.lbl-previewImage')}>
+      <InputGroup name="Preview Image Bake" label={t('editor:properties.portal.lbl-generateImage')}>
         <div style={{ display: 'flex', flexDirection: 'column' }}>
           <div style={{ width: 'auto', display: 'flex', flexDirection: 'row' }}>
             <Button
               style={{ width: '100%', fontSize: '11px', overflow: 'hidden' }}
               type="submit"
               onClick={() => {
-                bakeCubemap()
+                uploadEnvmap()
               }}
             >
               {t('editor:properties.portal.lbl-saveImage')}
@@ -204,10 +193,10 @@ export const PortalNodeEditor: EditorComponentType = (props) => {
                 updateCubeMapBake()
               }}
             >
-              {t('editor:properties.portal.lbl-previewImage')}
+              {t('editor:properties.portal.lbl-generateImage')}
             </Button>
           </div>
-          <ImagePreviewInput value={bufferUrl} />
+          <ImagePreviewInput value={state.previewImageURL.value ?? portalComponent.previewImageURL.value} />
         </div>
       </InputGroup>
       <InputGroup name="Spawn Position" label={t('editor:properties.portal.lbl-spawnPosition')}>

packages/editor/src/functions/uploadEnvMapBake.ts

@@ -39,7 +39,6 @@ import CubemapCapturer from '@etherealengine/engine/src/scene/classes/CubemapCap
 import {
   blurAndScaleImageData,
   convertCubemapToEquiImageData,
-  convertCubemapToImageData,
   convertImageDataToKTX2Blob
 } from '@etherealengine/engine/src/scene/classes/ImageUtils'
 import { EnvMapBakeComponent } from '@etherealengine/engine/src/scene/components/EnvMapBakeComponent'
@@ -101,12 +100,12 @@ export const uploadBPCEMBakeToServer = async (entity: Entity) => {
 
   if (isSceneEntity) Engine.instance.scene.environment = renderTarget.texture
 
-  const envmapImageData = convertCubemapToImageData(
+  const envmapImageData = convertCubemapToEquiImageData(
     EngineRenderer.instance.renderer,
     renderTarget.texture,
     bakeComponent.resolution,
     bakeComponent.resolution
-  )
+  ) as ImageData
 
   const envmap = await convertImageDataToKTX2Blob(envmapImageData)
 
@@ -142,12 +141,12 @@ export const uploadSceneBakeToServer = async () => {
 
   Engine.instance.scene.environment = renderTarget.texture
 
-  const envmapImageData = convertCubemapToImageData(
+  const envmapImageData = convertCubemapToEquiImageData(
     EngineRenderer.instance.renderer,
     renderTarget.texture,
     bakeComponent.resolution,
     bakeComponent.resolution
-  )
+  ) as ImageData
 
   const loadingScreenImageData = blurAndScaleImageData(
     envmapImageData,
@@ -182,27 +181,22 @@ export const uploadSceneBakeToServer = async () => {
 
 const resolution = 1024
 
-const previewCubemapCapturer = new CubemapCapturer(
-  EngineRenderer.instance.renderer,
-  Engine.instance.scene,
-  resolution / 8
-)
+const previewCubemapCapturer = new CubemapCapturer(EngineRenderer.instance.renderer, Engine.instance.scene, resolution)
 /**
  * Generates a low res cubemap at a specific position in the world for preview.
  *
  * @param position
  * @returns
  */
-export const getPreviewBakeTexture = async (position: Vector3) => {
+export const bakeEnvmapTexture = async (position: Vector3) => {
   const renderTarget = previewCubemapCapturer.update(position)
-  const imageBlob = (await convertCubemapToEquiImageData(
+  const bake = (await convertCubemapToEquiImageData(
     EngineRenderer.instance.renderer,
     renderTarget.texture,
-    resolution / 4,
-    resolution / 4,
-    true
-  )) as Blob
-  return imageBlob
+    resolution,
+    resolution
+  )) as ImageData
+  return bake
 }
 
 /**
@@ -211,12 +205,8 @@ export const getPreviewBakeTexture = async (position: Vector3) => {
  * @param position
  * @returns
  */
-const saveCubemapCapturer = new CubemapCapturer(EngineRenderer.instance.renderer, Engine.instance.scene, resolution)
-export const uploadCubemapBakeToServer = async (name: string, position: Vector3, res: number = resolution) => {
-  const renderTarget = saveCubemapCapturer.update(position)
-  const bake = await convertCubemapToImageData(EngineRenderer.instance.renderer, renderTarget.texture, res, res)
-
-  const blob = await convertImageDataToKTX2Blob(bake)
+export const uploadCubemapBakeToServer = async (name: string, data: ImageData) => {
+  const blob = await convertImageDataToKTX2Blob(data)
 
   if (!blob) return null!
 

packages/engine/src/scene/classes/CubemapCapturer.ts

@@ -70,4 +70,8 @@ export default class CubemapCapturer {
     this.renderer.autoClear = autoClear
     return this.cubeRenderTarget
   }
+
+  dispose() {
+    this.cubeRenderTarget.dispose()
+  }
 }

packages/engine/src/scene/classes/ImageUtils.ts

@@ -151,57 +151,6 @@ export const ScreenshotSettings = defineState({
 
 const ktx2write = new KTX2Encoder()
 
-export const convertCubemapToImageData = (
-  renderer: WebGLRenderer,
-  source: CubeTexture,
-  width: number,
-  height: number
-) => {
-  const scene = new Scene()
-  const material = new RawShaderMaterial({
-    uniforms: {
-      map: new Uniform(new CubeTexture())
-    },
-    vertexShader: vertexShader,
-    fragmentShader: fragmentShader,
-    side: DoubleSide,
-    transparent: true
-  })
-  const quad = new Mesh(new PlaneGeometry(1, 1), material)
-  scene.add(quad)
-  const camera = new OrthographicCamera(1 / -2, 1 / 2, 1 / 2, 1 / -2, -10000, 10000)
-
-  quad.scale.set(width, height, 1)
-  camera.left = width / 2
-  camera.right = width / -2
-  camera.top = height / -2
-  camera.bottom = height / 2
-  camera.updateProjectionMatrix()
-  const renderTarget = new WebGLRenderTarget(width, height, {
-    minFilter: LinearFilter,
-    magFilter: LinearFilter,
-    wrapS: ClampToEdgeWrapping,
-    wrapT: ClampToEdgeWrapping,
-    colorSpace: SRGBColorSpace,
-    format: RGBAFormat,
-    type: UnsignedByteType
-  })
-
-  const originalColorSpace = renderer.outputColorSpace
-  renderer.outputColorSpace = SRGBColorSpace
-  renderer.setRenderTarget(renderTarget)
-  quad.material.uniforms.map.value = source
-
-  renderer.render(scene, camera)
-  const pixels = new Uint8Array(4 * width * height)
-  renderer.readRenderTargetPixels(renderTarget, 0, 0, width, height, pixels)
-
-  renderer.setRenderTarget(null) // pass `null` to set canvas as render target
-  renderer.outputColorSpace = originalColorSpace
-
-  return new ImageData(new Uint8ClampedArray(pixels), width, height)
-}
-
 export const convertImageDataToKTX2Blob = async (imageData: ImageData) => {
   const ktx2texture = (await ktx2write.encode(imageData, getState(ScreenshotSettings).ktx2)) as ArrayBuffer
   return new Blob([ktx2texture])
@@ -260,47 +209,47 @@ export const convertCubemapToEquiImageData = (
   const camera = new OrthographicCamera(1 / -2, 1 / 2, 1 / 2, 1 / -2, -10000, 10000)
 
   quad.scale.set(width, height, 1)
-  camera.left = width / -2
-  camera.right = width / 2
-  camera.top = height / 2
-  camera.bottom = height / -2
+  camera.left = width / 2
+  camera.right = width / -2
+  camera.top = height / -2
+  camera.bottom = height / 2
   camera.updateProjectionMatrix()
   const renderTarget = new WebGLRenderTarget(width, height, {
     minFilter: LinearFilter,
     magFilter: LinearFilter,
     wrapS: ClampToEdgeWrapping,
     wrapT: ClampToEdgeWrapping,
+    colorSpace: SRGBColorSpace,
     format: RGBAFormat,
     type: UnsignedByteType
   })
 
+  const originalColorSpace = renderer.outputColorSpace
+  renderer.outputColorSpace = SRGBColorSpace
   renderer.setRenderTarget(renderTarget)
   quad.material.uniforms.map.value = source
   renderer.render(scene, camera)
   const pixels = new Uint8Array(4 * width * height)
   renderer.readRenderTargetPixels(renderTarget, 0, 0, width, height, pixels)
-  const imageData = new ImageData(new Uint8ClampedArray(pixels), width, height)
   renderer.setRenderTarget(null) // pass `null` to set canvas as render target
+  renderer.outputColorSpace = originalColorSpace
 
-  if (returnAsBlob) {
-    const canvas = document.createElement('canvas')
-    const ctx = canvas.getContext('2d') as CanvasRenderingContext2D
-    canvas.width = width
-    canvas.height = height
-    ctx.putImageData(imageData, 0, 0)
-    return new Promise<Blob | null>((resolve) => canvas.toBlob(resolve))
-  }
-
-  /* const writer = new GLTFWriter()
-  writer.processSampler(source)
+  const imageData = new ImageData(new Uint8ClampedArray(pixels), width, height)
 
-  const g = new GLTF
-  writer.write(new Object3D())
-*/
+  if (returnAsBlob) return imageDataToBlob(imageData)
 
   return imageData
 }
 
+export const imageDataToBlob = (imageData: ImageData): Promise<Blob | null> => {
+  const canvas = document.createElement('canvas')
+  const ctx = canvas.getContext('2d') as CanvasRenderingContext2D
+  canvas.width = imageData.width
+  canvas.height = imageData.height
+  ctx.putImageData(imageData, 0, 0)
+  return new Promise<Blob | null>((resolve) => canvas.toBlob(resolve))
+}
+
 //convert Equirectangular map to WebGlCubeRenderTarget
 export const convertEquiToCubemap = (renderer: WebGLRenderer, source: Texture, size: number): WebGLCubeRenderTarget => {
   const convertScene = new Scene()