Releasing soon (#183)

.github/workflows/next-deploy.yml

@@ -3,7 +3,7 @@ env:
   VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
   VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
   ALIASES: ${{ vars.ALIASES }}
-  MAIN_MENU_LINKS: ${{ secrets.MAIN_MENU_LINKS }}
+  MAIN_MENU_LINKS: ${{ vars.MAIN_MENU_LINKS }}
 on:
   push:
     branches:

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Release
 env:
   VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
   VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
-  MAIN_MENU_LINKS: ${{ secrets.MAIN_MENU_LINKS }}
+  MAIN_MENU_LINKS: ${{ vars.MAIN_MENU_LINKS }}
 on:
   push:
     branches: [release]

README.MD

@@ -141,7 +141,31 @@ Single player specific:
 - `?singleplayer=1` - Create empty world on load. Nothing will be saved
 - `?version=<version>` - Set the version for the singleplayer world (when used with `?singleplayer=1`)
 - `?noSave=true` - Disable auto save on unload / disconnect / export whenever a world is loaded. Only manual save with `/save` command will work.
-- `?map=<map_url>` - Load the map from ZIP. You can use any url, but it must be CORS enabled.
+- `?map=<map_url>` - Load the map from ZIP. You can use any url, but it must be **CORS enabled**.
+- `?mapDir=<index_file_url>` - Load the map from a file descriptor. It's recommended and the fastest way to load world but requires additional setup. The file must be in the following format:
+
+```json
+{
+    "baseUrl": "<url>",
+    "index": {
+        "level.dat": null,
+        "region": {
+            "r.-1.-1.mca": null,
+            "r.-1.0.mca": null,
+            "r.0.-1.mca": null,
+            "r.0.0.mca": null,
+        }
+    }
+}
+```
+
+Note that `mapDir` also accepts base64 encoded JSON like so:
+`?mapDir=data:application/json;base64,...` where `...` is the base64 encoded JSON of the index file.
+In this case you must use `?mapDirBaseUrl` to specify the base URL to fetch the files from index.
+
+- `?mapDirBaseUrl` - See above.
+
+<!-- - `?mapDirGuess=<base_url>` - Load the map from the provided URL and paths will be guessed with a few additional fetch requests. -->
 
 General:
 

package.json

@@ -68,7 +68,7 @@
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "express": "^4.18.2",
     "filesize": "^10.0.12",
-    "flying-squid": "npm:@zardoy/flying-squid@^0.0.34",
+    "flying-squid": "npm:@zardoy/flying-squid@^0.0.35",
     "fs-extra": "^11.1.1",
     "google-drive-browserfs": "github:zardoy/browserfs#google-drive",
     "jszip": "^3.10.1",

prismarine-viewer/viewer/lib/entities.js

@@ -15,7 +15,7 @@ import { WalkingGeneralSwing } from './entity/animations'
 import externalTexturesJson from './entity/externalTextures.json'
 import { disposeObject } from './threeJsUtils'
 
-export const TWEEN_DURATION = 50 // todo should be 100
+export const TWEEN_DURATION = 120
 
 /**
  * @param {string} username

prismarine-viewer/viewer/lib/mesher/models.ts

@@ -354,6 +354,8 @@ function renderElement (world: World, cursor: Vec3, element: BlockElement, doAO:
   }
 }
 
+const makeLooseObj = <T extends string> (obj: Record<T, any>) => obj
+
 const invisibleBlocks = new Set(['air', 'cave_air', 'void_air', 'barrier'])
 
 const isBlockWaterlogged = (block: Block) => block.getProperties().waterlogged === true || block.getProperties().waterlogged === 'true'
@@ -363,7 +365,7 @@ let erroredBlockModel: BlockModelPartsResolved
 export function getSectionGeometry (sx, sy, sz, world: World) {
   let delayedRender = [] as Array<() => void>
 
-  const attr = {
+  const attr = makeLooseObj({
     sx: sx + 8,
     sy: sy + 8,
     sz: sz + 8,
@@ -379,14 +381,24 @@ export function getSectionGeometry (sx, sy, sz, world: World) {
     tiles: {},
     // todo this can be removed here
     signs: {},
+    highestBlocks: {},
     hadErrors: false
-  } as Record<string, any>
+  } as Record<string, any>)
 
   const cursor = new Vec3(0, 0, 0)
   for (cursor.y = sy; cursor.y < sy + 16; cursor.y++) {
     for (cursor.z = sz; cursor.z < sz + 16; cursor.z++) {
       for (cursor.x = sx; cursor.x < sx + 16; cursor.x++) {
         const block = world.getBlock(cursor)!
+        if (!invisibleBlocks.has(block.name)) {
+          const highest = attr.highestBlocks[`${cursor.x},${cursor.z}`]
+          if (!highest || highest.y < cursor.y) {
+            attr.highestBlocks[`${cursor.x},${cursor.z}`] = {
+              y: cursor.y,
+              name: block.name
+            }
+          }
+        }
         if (invisibleBlocks.has(block.name)) continue
         if (block.name.includes('_sign') || block.name === 'sign') {
           const key = `${cursor.x},${cursor.y},${cursor.z}`

prismarine-viewer/viewer/lib/viewer.ts

@@ -170,6 +170,8 @@ export class Viewer {
     })
   }
 
+  addChunksBatchWaitTime = 200
+
   // todo type
   listen (emitter: EventEmitter) {
     emitter.on('entity', (e) => {
@@ -180,9 +182,26 @@ export class Viewer {
       // this.updatePrimitive(p)
     })
 
+    let currentLoadChunkBatch = null as {
+      timeout
+      data
+    } | null
     emitter.on('loadChunk', ({ x, z, chunk, worldConfig, isLightUpdate }) => {
       this.world.worldConfig = worldConfig
-      this.addColumn(x, z, chunk, isLightUpdate)
+      if (!currentLoadChunkBatch) {
+        // add a setting to use debounce instead
+        currentLoadChunkBatch = {
+          data: [],
+          timeout: setTimeout(() => {
+            for (const args of currentLoadChunkBatch!.data) {
+              //@ts-expect-error
+              this.addColumn(...args)
+            }
+            currentLoadChunkBatch = null
+          }, this.addChunksBatchWaitTime)
+        }
+      }
+      currentLoadChunkBatch.data.push([x, z, chunk, isLightUpdate])
     })
     // todo remove and use other architecture instead so data flow is clear
     emitter.on('blockEntities', (blockEntities) => {

prismarine-viewer/viewer/lib/worldDataEmitter.ts

@@ -19,6 +19,7 @@ export class WorldDataEmitter extends EventEmitter {
   private readonly lastPos: Vec3
   private eventListeners: Record<string, any> = {}
   private readonly emitter: WorldDataEmitter
+  keepChunksDistance = 0
 
   constructor (public world: typeof __type_bot['world'], public viewDistance: number, position: Vec3 = new Vec3(0, 0, 0)) {
     super()
@@ -150,6 +151,9 @@ export class WorldDataEmitter extends EventEmitter {
     }
   }
 
+  // debugGotChunkLatency = [] as number[]
+  // lastTime = 0
+
   async loadChunk (pos: ChunkPos, isLightUpdate = false) {
     const [botX, botZ] = chunkPos(this.lastPos)
     const dx = Math.abs(botX - Math.floor(pos.x / 16))
@@ -158,6 +162,9 @@ export class WorldDataEmitter extends EventEmitter {
       // eslint-disable-next-line @typescript-eslint/await-thenable -- todo allow to use async world provider but not sure if needed
       const column = await this.world.getColumnAt(pos['y'] ? pos as Vec3 : new Vec3(pos.x, 0, pos.z))
       if (column) {
+        // const latency = Math.floor(performance.now() - this.lastTime)
+        // this.debugGotChunkLatency.push(latency)
+        // this.lastTime = performance.now()
         // todo optimize toJson data, make it clear why it is used
         const chunk = column.toJson()
         // TODO: blockEntities
@@ -191,14 +198,14 @@ export class WorldDataEmitter extends EventEmitter {
     const [botX, botZ] = chunkPos(pos)
     if (lastX !== botX || lastZ !== botZ || force) {
       this.emitter.emit('chunkPosUpdate', { pos })
-      const newView = new ViewRect(botX, botZ, this.viewDistance)
+      const newViewToUnload = new ViewRect(botX, botZ, this.viewDistance + this.keepChunksDistance)
       const chunksToUnload: Vec3[] = []
       for (const coords of Object.keys(this.loadedChunks)) {
         const x = parseInt(coords.split(',')[0], 10)
         const z = parseInt(coords.split(',')[1], 10)
         const p = new Vec3(x, 0, z)
         const [chunkX, chunkZ] = chunkPos(p)
-        if (!newView.contains(chunkX, chunkZ)) {
+        if (!newViewToUnload.contains(chunkX, chunkZ)) {
           chunksToUnload.push(p)
         }
       }

prismarine-viewer/viewer/lib/worldrendererCommon.ts

@@ -75,6 +75,7 @@ export abstract class WorldRendererCommon<WorkerSend = any, WorkerReceive = any>
   handleResize = () => { }
   mesherConfig = defaultMesherConfig
   camera: THREE.PerspectiveCamera
+  highestBlocks: Record<string, { y: number, name: string }> = {}
   blockstatesModels: any
   customBlockStates: Record<string, any> | undefined
   customModels: Record<string, any> | undefined
@@ -109,7 +110,15 @@ export abstract class WorldRendererCommon<WorkerSend = any, WorkerReceive = any>
       const handleMessage = (data) => {
         if (!this.active) return
         this.handleWorkerMessage(data)
-        if (data.type === 'sectionFinished') {
+        if (data.type === 'geometry') {
+          for (const key in data.geometry.highestBlocks) {
+            const highest = data.geometry.highestBlocks[key]
+            if (!this.highestBlocks[key] || this.highestBlocks[key].y < highest.y) {
+              this.highestBlocks[key] = highest
+            }
+          }
+        }
+        if (data.type === 'sectionFinished') { // on after load & unload section
           if (!this.sectionsOutstanding.get(data.key)) throw new Error(`sectionFinished event for non-outstanding section ${data.key}`)
           this.sectionsOutstanding.set(data.key, this.sectionsOutstanding.get(data.key)! - 1)
           if (this.sectionsOutstanding.get(data.key) === 0) this.sectionsOutstanding.delete(data.key)
@@ -302,6 +311,19 @@ export abstract class WorldRendererCommon<WorkerSend = any, WorkerReceive = any>
     }
     this.allChunksFinished = Object.keys(this.finishedChunks).length === this.chunksLength
     delete this.finishedChunks[`${x},${z}`]
+    for (let y = this.worldConfig.minY; y < this.worldConfig.worldHeight; y += 16) {
+      this.setSectionDirty(new Vec3(x, y, z), false)
+    }
+    // remove from highestBlocks
+    const startX = Math.floor(x / 16) * 16
+    const startZ = Math.floor(z / 16) * 16
+    const endX = Math.ceil((x + 1) / 16) * 16
+    const endZ = Math.ceil((z + 1) / 16) * 16
+    for (let x = startX; x < endX; x += 16) {
+      for (let z = startZ; z < endZ; z += 16) {
+        delete this.highestBlocks[`${x},${z}`]
+      }
+    }
   }
 
   setBlockStateId (pos: Vec3, stateId: number) {
@@ -320,7 +342,7 @@ export abstract class WorldRendererCommon<WorkerSend = any, WorkerReceive = any>
   queueAwaited = false
   messagesQueue = {} as { [workerIndex: string]: any[] }
 
-  setSectionDirty (pos: Vec3, value = true) {
+  setSectionDirty (pos: Vec3, value = true) { // value false is used for unloading chunks
     if (this.viewDistance === -1) throw new Error('viewDistance not set')
     this.allChunksFinished = false
     const distance = this.getDistance(pos)

prismarine-viewer/viewer/lib/worldrendererThree.ts

@@ -66,6 +66,7 @@ export class WorldRendererThree extends WorldRendererCommon {
     }
   }
 
+  // debugRecomputedDeletedObjects = 0
   handleWorkerMessage (data: any): void {
     if (data.type !== 'geometry') return
     let object: THREE.Object3D = this.sectionObjects[data.key]
@@ -78,6 +79,10 @@ export class WorldRendererThree extends WorldRendererCommon {
     const chunkCoords = data.key.split(',')
     if (!this.loadedChunks[chunkCoords[0] + ',' + chunkCoords[2]] || !data.geometry.positions.length || !this.active) return
 
+    // if (object) {
+    //   this.debugRecomputedDeletedObjects++
+    // }
+
     // if (!this.initialChunksLoad && this.enableChunksLoadDelay) {
     //   const newPromise = new Promise(resolve => {
     //     if (this.droppedFpsPercentage > 0.5) {

rsbuild.config.ts

@@ -49,11 +49,12 @@ export default defineConfig({
             'minecraft-protocol$': 'minecraft-protocol/src/index.js',
             'buffer$': 'buffer',
             // avoid bundling, not used on client side
-            'prismarine-auth': './src/shims/empty.ts',
+            'prismarine-auth': './src/shims/prismarineAuthReplacement.ts',
             perf_hooks: './src/shims/perf_hooks_replacement.js',
             crypto: './src/shims/crypto.js',
             dns: './src/shims/dns.js',
             yggdrasil: './src/shims/yggdrasilReplacement.ts',
+            'three$': 'three/src/Three.js'
         },
         entry: {
             index: './src/index.ts',
@@ -182,9 +183,10 @@ export default defineConfig({
             ]
         }
     },
-    performance: {
-        // bundleAnalyze: {
-        //     analyzerMode: 'json',
-        // },
-    },
+    // performance: {
+    //     bundleAnalyze: {
+    //         analyzerMode: 'json',
+    //         reportFilename: 'report.json',
+    //     },
+    // },
 })

server.js

@@ -17,6 +17,7 @@ const app = express()
 
 const isProd = process.argv.includes('--prod')
 app.use(compression())
+// app.use(cors())
 app.use(netApi({ allowOrigin: '*' }))
 if (!isProd) {
   app.use('/sounds', express.static(path.join(__dirname, './generated/sounds/')))