Simplify Static Resources (#9061)

* simplify static resource parsing of scenes and uploads

* add download button in file browser input

* fix

Author: HexaField

.env.local.default

@@ -93,7 +93,6 @@ LOCAL_STORAGE_PROVIDER_PORT=8642
 GOOGLE_ANALYTICS_TRACKING_ID=
 HUB_ENDPOINT=https://etherealengine.io
 INSTANCESERVER_UNREACHABLE_TIMEOUT_SECONDS=10
-CLONE_STATIC_RESOURCES=false
 
 
 MATCHMAKER_EMULATION_MODE=true

packages/editor/src/components/inputs/ArrayInputGroup.tsx

@@ -23,8 +23,8 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
 Ethereal Engine. All Rights Reserved.
 */
 
+import Icon from '@etherealengine/ui/src/primitives/mui/Icon'
 import AddIcon from '@mui/icons-material/Add'
-import DeleteIcon from '@mui/icons-material/Delete'
 import IconButton from '@mui/material/IconButton'
 import React from 'react'
 import styles from './ArrayInputGroup.module.scss'
@@ -105,7 +105,7 @@ const ArrayInputGroup = ({
               padding: 0
             }}
           >
-            <AddIcon sx={{ color: 'primary.contrastText' }} />
+            <AddIcon sx={{ color: 'var(--textColor)' }} />
           </IconButton>
         </InputGroup>
         {values &&
@@ -125,7 +125,7 @@ const ArrayInputGroup = ({
                 }}
                 onClick={() => deleteInput(index + 1)}
               >
-                <DeleteIcon sx={{ color: 'primary.contrastText' }} />
+                <Icon type="Delete" style={{ color: 'var(--textColor)' }} />
               </IconButton>
             </InputGroup>
           ))}

packages/editor/src/components/inputs/FileBrowserInput.tsx

@@ -23,9 +23,12 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
 Ethereal Engine. All Rights Reserved.
 */
 
+import IconButton from '@mui/material/IconButton'
 import React from 'react'
 import { useDrop } from 'react-dnd'
 
+import config from '@etherealengine/common/src/config'
+import Icon from '@etherealengine/ui/src/primitives/mui/Icon'
 import { ItemTypes } from '../../constants/AssetTypes'
 import useUpload from '../assets/useUpload'
 import { ControlledStringInput, StringInputProps } from './StringInput'
@@ -54,6 +57,32 @@ export function FileBrowserInput({
   }
   const onUpload = useUpload(uploadOptions)
 
+  // todo fix for invalid URLs
+  const assetIsExternal = value && !value?.includes(config.client.fileServer)
+  const uploadExternalAsset = () => {
+    onUpload([
+      {
+        isFile: true,
+        name: value?.split('/').pop(),
+        file: async (onSuccess, onFail) => {
+          try {
+            const asset = await fetch(value!)
+            const blob = await asset.blob()
+            const file = new File([blob], value!.split('/').pop()!)
+            onSuccess(file)
+          } catch (error) {
+            if (onFail) onFail(error)
+            else throw error
+          }
+        }
+      } as Partial<FileSystemFileEntry>
+    ] as any).then((assets) => {
+      if (assets) {
+        onChange?.(assets[0])
+      }
+    })
+  }
+
   const [{ canDrop, isOver }, dropRef] = useDrop({
     accept: [...acceptDropItems, ItemTypes.File],
     async drop(item: any, monitor) {
@@ -73,9 +102,7 @@ export function FileBrowserInput({
 
         onUpload(entries).then((assets) => {
           if (assets) {
-            for (let index = 0; index < assets.length; index++) {
-              onChange?.(assets[index])
-            }
+            onChange?.(assets[0])
           }
         })
       }
@@ -96,6 +123,17 @@ export function FileBrowserInput({
         canDrop={isOver && canDrop}
         {...rest}
       />
+      {assetIsExternal && (
+        <IconButton
+          disableRipple
+          style={{
+            padding: 0
+          }}
+          onClick={uploadExternalAsset}
+        >
+          <Icon type="Download" style={{ color: 'var(--textColor)' }} />
+        </IconButton>
+      )}
     </>
   )
 }

packages/server-core/src/appconfig.ts

@@ -145,8 +145,6 @@ const server = {
   corsServerPort: process.env.CORS_SERVER_PORT!,
   storageProvider: process.env.STORAGE_PROVIDER!,
   storageProviderExternalEndpoint: process.env.STORAGE_PROVIDER_EXTERNAL_ENDPOINT!,
-  cloneProjectStaticResources:
-    typeof process.env.CLONE_STATIC_RESOURCES === 'undefined' ? true : process.env.CLONE_STATIC_RESOURCES === 'true',
   gaTrackingId: process.env.GOOGLE_ANALYTICS_TRACKING_ID!,
   hub: {
     endpoint: process.env.HUB_ENDPOINT!

packages/server-core/src/media/static-resource/static-resource-helper.test.ts

@@ -25,19 +25,10 @@ Ethereal Engine. All Rights Reserved.
 
 import appRootPath from 'app-root-path'
 import assert from 'assert'
-import fs from 'fs'
 import path from 'path'
 
-import { destroyEngine } from '@etherealengine/engine/src/ecs/classes/Engine'
-
-import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema'
-import { Paginated } from '@feathersjs/feathers'
-import { Application } from '../../../declarations'
 import { mockFetch, restoreFetch } from '../../../tests/util/mockFetch'
-import { createFeathersKoaApp } from '../../createApp'
-import { getCachedURL } from '../storageprovider/getCachedURL'
-import { getStorageProvider } from '../storageprovider/storageprovider'
-import { addAssetFromProject, downloadResourceAndMetadata } from './static-resource-helper'
+import { downloadResourceAndMetadata } from './static-resource-helper'
 
 describe('static-resource-helper', () => {
   before(() => {
@@ -91,201 +82,3 @@ describe('static-resource-helper', () => {
     })
   })
 })
-
-const testProject = 'test-project'
-
-describe('audio-upload', () => {
-  let app: Application
-
-  before(async () => {
-    app = createFeathersKoaApp()
-    await app.setup()
-    const url = 'https://test.com/projects/default-project/assets/test.mp3'
-    const url2 = getCachedURL('/projects/default-project/assets/test.mp3', getStorageProvider().cacheDomain)
-    mockFetch({
-      [url]: {
-        contentType: 'audio/mpeg',
-        response: fs.readFileSync(
-          path.join(appRootPath.path, '/packages/projects/default-project/assets/SampleAudio.mp3')
-        )
-      },
-      [url2]: {
-        contentType: 'audio/mpeg',
-        response: fs.readFileSync(
-          path.join(appRootPath.path, '/packages/projects/default-project/assets/SampleAudio.mp3')
-        )
-      }
-    })
-  })
-
-  afterEach(async () => {
-    const storageProvider = getStorageProvider()
-    if (await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))
-      await storageProvider.deleteResources(['static-resources/test-project/test.mp3'])
-
-    const existingResource = (await app.service(staticResourcePath).find({
-      query: {
-        mimeType: 'audio/mpeg'
-      },
-      paginate: false
-    })) as StaticResourceType[]
-    await Promise.all(existingResource.map((resource) => app.service(staticResourcePath).remove(resource.id)))
-  })
-
-  after(() => {
-    restoreFetch()
-    return destroyEngine()
-  })
-
-  describe('addAssetFromProject', () => {
-    it('should download audio asset as a new static resource from external url when forced to download', async () => {
-      const storageProvider = getStorageProvider()
-      const url = 'https://test.com/projects/default-project/assets/test.mp3'
-
-      const staticResource = await addAssetFromProject(app, url, testProject, true)
-
-      assert(staticResource.id)
-      assert.equal(staticResource.url, getCachedURL(staticResource.key!, storageProvider.cacheDomain))
-      assert.equal(staticResource.key, 'static-resources/test-project/test.mp3')
-      assert.equal(staticResource.mimeType, 'audio/mpeg')
-      assert.equal(staticResource.project, testProject)
-
-      assert(await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))
-
-      const file = await storageProvider.getObject(staticResource.key!)
-      assert.equal(file.ContentType, 'audio/mpeg')
-    })
-
-    it('should link audio asset as a new static resource from external url when not forced to download', async () => {
-      const storageProvider = getStorageProvider()
-      const url = 'https://test.com/projects/default-project/assets/test.mp3'
-
-      const staticResource = await addAssetFromProject(app, url, testProject, false)
-
-      assert(staticResource.id)
-      assert.equal(staticResource.url, 'https://test.com/projects/default-project/assets/test.mp3')
-      assert.equal(staticResource.key, 'https://test.com/projects/default-project/assets/test.mp3')
-      assert.equal(staticResource.mimeType, 'audio/mpeg')
-      assert.equal(staticResource.project, testProject)
-
-      const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
-      assert(!fileExists)
-    })
-
-    it('should download audio asset as a new static resource from another project', async () => {
-      const storageProvider = getStorageProvider()
-      const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)
-
-      const staticResource = await addAssetFromProject(app, url, testProject, true)
-
-      assert.equal(staticResource.key, 'static-resources/test-project/test.mp3')
-      assert.equal(staticResource.url, getCachedURL(staticResource.key!, storageProvider.cacheDomain))
-      assert.equal(staticResource.mimeType, 'audio/mpeg')
-      assert.equal(staticResource.project, testProject)
-
-      assert(await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))
-
-      const file = await storageProvider.getObject(staticResource.key!)
-      assert.equal(file.ContentType, 'audio/mpeg')
-    })
-
-    it('should link audio asset as a new static resource from another project', async () => {
-      const storageProvider = getStorageProvider()
-      const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)
-
-      const staticResource = await addAssetFromProject(app, url, testProject, false)
-
-      assert.equal(staticResource.key, url)
-      assert.equal(staticResource.url, url)
-      assert.equal(staticResource.mimeType, 'audio/mpeg')
-      assert.equal(staticResource.project, testProject)
-
-      // should not exist under static resources
-      const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
-      assert(!fileExists)
-    })
-
-    it('should link audio asset as a new static resource from url if from the same project', async () => {
-      const storageProvider = getStorageProvider()
-      const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)
-
-      const staticResource = await addAssetFromProject(app, url, 'default-project', false)
-
-      assert.equal(staticResource.key, 'projects/default-project/assets/test.mp3')
-      assert.equal(staticResource.url, url)
-      assert.equal(staticResource.mimeType, 'audio/mpeg')
-      assert.equal(staticResource.project, 'default-project')
-
-      // should not exist under static resources
-      const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
-      assert(!fileExists)
-    })
-
-    it('should return existing static resource with the same hash and project', async () => {
-      const storageProvider = getStorageProvider()
-      const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)
-
-      const response = await addAssetFromProject(app, url, 'default-project', false)
-      const response2 = await addAssetFromProject(app, url, 'default-project', false)
-
-      const staticResources = (await app.service(staticResourcePath).find({
-        query: {
-          url
-        }
-      })) as Paginated<StaticResourceType>
-
-      assert.equal(staticResources.data.length, 1)
-      assert.equal(response.id, response2.id)
-      assert.equal(response.url, response2.url)
-      assert.equal(response.key, response2.key)
-    })
-
-    it('should return new static resource with the same hash exists in another project when forcing download', async () => {
-      const storageProvider = getStorageProvider()
-      const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)
-
-      const response = await addAssetFromProject(app, url, 'default-project', true)
-      const response2 = await addAssetFromProject(app, url, 'test-project', true)
-
-      const staticResources = (await app.service(staticResourcePath).find({
-        query: {
-          url: response.url
-        }
-      })) as Paginated<StaticResourceType>
-      assert.equal(staticResources.data.length, 1)
-
-      const staticResources2 = (await app.service(staticResourcePath).find({
-        query: {
-          url: response2.url
-        }
-      })) as Paginated<StaticResourceType>
-      assert.equal(staticResources2.data.length, 1)
-
-      assert.notEqual(response.id, response2.id)
-      assert.notEqual(response.url, response2.url)
-      assert.notEqual(response.key, response2.key)
-      assert.equal(response.hash, response2.hash)
-    })
-
-    it('should different static resource with the same url and hash if it exists in another project when not forcing download', async () => {
-      const storageProvider = getStorageProvider()
-      const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)
-
-      const response = await addAssetFromProject(app, url, 'default-project', false)
-      const response2 = await addAssetFromProject(app, url, 'test-project', false)
-
-      const staticResources = (await app.service(staticResourcePath).find({
-        query: {
-          url: response.url
-        }
-      })) as Paginated<StaticResourceType>
-      assert.equal(staticResources.data.length, 2)
-
-      assert(response.id !== response2.id)
-      assert.equal(response.url, response2.url)
-      // key is different as it is a different project
-      assert.notEqual(response.key, response2.key)
-      assert.equal(response.hash, response2.hash)
-    })
-  })
-})