Skip to content

Support creating assets from PLY files larger than 2GB #228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
comoc wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Support creating assets from PLY files larger than 2GB #228

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
79c6ad4
Support creating assets from PLY files larger than 2GB
comoc Jun 29, 2026
90fef09
Read PLY body via a Burst job instead of a direct-call function pointer
comoc Jun 29, 2026
c6d02f8
Address review: avoid leaking the splat array on read failure
comoc Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions package/Editor/GaussianSplatAssetCreator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ public int Compare((ulong, int) a, (ulong, int) b)
}
}

static void ReorderMorton(NativeArray<InputSplatData> splatData, float3 boundsMin, float3 boundsMax)
static unsafe void ReorderMorton(NativeArray<InputSplatData> splatData, float3 boundsMin, float3 boundsMax)
{
ReorderMortonJob order = new ReorderMortonJob
{
Expand All @@ -420,9 +420,17 @@ static void ReorderMorton(NativeArray<InputSplatData> splatData, float3 boundsMi
order.Schedule(splatData.Length, 4096).Complete();
order.m_Order.Sort(new OrderComparer());

NativeArray<InputSplatData> copy = new(order.m_SplatData, Allocator.TempJob);
// Copy the source data, then gather it back in the new order. Use an explicit long-sized MemCpy
// instead of the NativeArray copy constructor / NativeArray.Copy: their internal size is computed as
// 'length * sizeof' in int, which overflows once the array exceeds 2GB (>8.6M splats * 248 bytes) and
// crashes inside memcpy.
NativeArray<InputSplatData> copy = new(splatData.Length, Allocator.TempJob, NativeArrayOptions.UninitializedMemory);
UnsafeUtility.MemCpy(copy.GetUnsafePtr(), splatData.GetUnsafeReadOnlyPtr(),
(long)splatData.Length * UnsafeUtility.SizeOf<InputSplatData>());
InputSplatData* copyPtr = (InputSplatData*)copy.GetUnsafeReadOnlyPtr();
InputSplatData* dstPtr = (InputSplatData*)splatData.GetUnsafePtr();
for (int i = 0; i < copy.Length; ++i)
order.m_SplatData[i] = copy[order.m_Order[i].Item2];
dstPtr[i] = copyPtr[order.m_Order[i].Item2];
copy.Dispose();

order.m_Order.Dispose();
Expand Down
82 changes: 67 additions & 15 deletions package/Editor/Utils/GaussianFileReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,7 @@ public static unsafe void ReadFile(string filePath, out NativeArray<InputSplatDa
{
if (isPLY(filePath))
{
NativeArray<byte> plyRawData;
List<(string, PLYFileReader.ElementType)> attributes;
PLYFileReader.ReadFile(filePath, out var splatCount, out var vertexStride, out attributes, out plyRawData);
string attrError = CheckPLYAttributes(attributes);
if (!string.IsNullOrEmpty(attrError))
throw new IOException($"PLY file is probably not a Gaussian Splat file? Missing properties: {attrError}");
splats = PLYDataToSplats(plyRawData, splatCount, vertexStride, attributes);
ReorderSHs(splatCount, (float*)splats.GetUnsafePtr());
LinearizeData(splats);
ReadPLYFile(filePath, out splats);
return;
}
if (isSPZ(filePath))
Expand All @@ -65,6 +57,65 @@ public static unsafe void ReadFile(string filePath, out NativeArray<InputSplatDa
throw new IOException($"File {filePath} is not a supported format");
}

static unsafe void ReadPLYFile(string filePath, out NativeArray<InputSplatData> splats)
{
using var fs = PLYFileReader.OpenAndReadHeader(filePath, out var splatCount, out var vertexStride, out var attributes);
string attrError = CheckPLYAttributes(attributes);
if (!string.IsNullOrEmpty(attrError))
throw new IOException($"PLY file is probably not a Gaussian Splat file? Missing properties: {attrError}");

// The destination splat array can be larger than 2GB in total (its element count still fits in an int),
// but the raw PLY body can exceed the ~2GB NativeArray<byte> size limit. So read the body in batches into
// a small reusable buffer and convert each batch straight into the destination array.
NativeArray<int> srcOffsets = BuildSrcOffsets(attributes);
int dstStride = UnsafeUtility.SizeOf<InputSplatData>();
splats = new NativeArray<InputSplatData>(splatCount, Allocator.Persistent);

const int kBatchSplats = 1024 * 1024;
int batchSplats = math.min(kBatchSplats, math.max(1, splatCount));
NativeArray<byte> rawBatch = new(batchSplats * vertexStride, Allocator.Persistent);
try
{
byte* dstBase = (byte*)splats.GetUnsafePtr();
int* srcOffPtr = (int*)srcOffsets.GetUnsafeReadOnlyPtr();
byte* rawPtr = (byte*)rawBatch.GetUnsafeReadOnlyPtr();

int splatIndex = 0;
while (splatIndex < splatCount)
{
int thisBatch = math.min(batchSplats, splatCount - splatIndex);
int bytesToRead = thisBatch * vertexStride;
int got = ReadExactly(fs, rawBatch, bytesToRead);
if (got != bytesToRead)
throw new IOException($"PLY {filePath} read error, expected {bytesToRead} data bytes got {got}");
ReorderPLYData(thisBatch, rawPtr, vertexStride, dstBase + (long)splatIndex * dstStride, dstStride, srcOffPtr);
splatIndex += thisBatch;
}
}
finally
{
rawBatch.Dispose();
srcOffsets.Dispose();
}
Comment thread
comoc marked this conversation as resolved.

ReorderSHs(splatCount, (float*)splats.GetUnsafePtr());
LinearizeData(splats);
}

// Reads exactly 'count' bytes from the stream into the start of 'buffer' (Stream.Read may return short reads).
static int ReadExactly(Stream fs, NativeArray<byte> buffer, int count)
{
int total = 0;
while (total < count)
{
int n = fs.Read(buffer.GetSubArray(total, count - total));
if (n <= 0)
break;
total += n;
}
return total;
}

static bool isPLY(string filePath) => filePath.EndsWith(".ply", true, CultureInfo.InvariantCulture);
static bool isSPZ(string filePath) => filePath.EndsWith(".spz", true, CultureInfo.InvariantCulture);

Expand All @@ -77,7 +128,9 @@ static string CheckPLYAttributes(List<(string, PLYFileReader.ElementType)> attri
return string.Join(",", missing);
}

static unsafe NativeArray<InputSplatData> PLYDataToSplats(NativeArray<byte> input, int count, int stride, List<(string, PLYFileReader.ElementType)> attributes)
// Builds, for each float field of InputSplatData, the byte offset of the matching attribute inside a PLY
// vertex record (or -1 if the attribute is absent). Returned array is Persistent; caller disposes it.
static NativeArray<int> BuildSrcOffsets(List<(string, PLYFileReader.ElementType)> attributes)
{
NativeArray<int> fileAttrOffsets = new NativeArray<int>(attributes.Count, Allocator.Temp);
int offset = 0;
Expand Down Expand Up @@ -154,17 +207,16 @@ static unsafe NativeArray<InputSplatData> PLYDataToSplats(NativeArray<byte> inpu
"rot_3",
};
Assert.AreEqual(UnsafeUtility.SizeOf<InputSplatData>() / 4, splatAttributes.Length);
NativeArray<int> srcOffsets = new NativeArray<int>(splatAttributes.Length, Allocator.Temp);
NativeArray<int> srcOffsets = new NativeArray<int>(splatAttributes.Length, Allocator.Persistent);
for (int ai = 0; ai < splatAttributes.Length; ai++)
{
int attrIndex = attributes.IndexOf((splatAttributes[ai], PLYFileReader.ElementType.Float));
int attrOffset = attrIndex >= 0 ? fileAttrOffsets[attrIndex] : -1;
srcOffsets[ai] = attrOffset;
}

NativeArray<InputSplatData> dst = new NativeArray<InputSplatData>(count, Allocator.Persistent);
ReorderPLYData(count, (byte*)input.GetUnsafeReadOnlyPtr(), stride, (byte*)dst.GetUnsafePtr(), UnsafeUtility.SizeOf<InputSplatData>(), (int*)srcOffsets.GetUnsafeReadOnlyPtr());
return dst;

fileAttrOffsets.Dispose();
return srcOffsets;
}

[BurstCompile]
Expand Down
34 changes: 18 additions & 16 deletions package/Editor/Utils/PLYFileReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using System.IO;
using System.Linq;
using System.Text;
using Unity.Collections;

namespace GaussianSplatting.Editor.Utils
{
Expand All @@ -22,12 +21,26 @@ public static void ReadFileHeader(string filePath, out int vertexCount, out int
ReadHeaderImpl(filePath, out vertexCount, out vertexStride, out attrs, fs);
}

static void ReadHeaderImpl(string filePath, out int vertexCount, out int vertexStride, out List<(string, ElementType)> attrs, FileStream fs)
// Opens the file, parses the header, and returns the stream positioned at the start of the binary
// vertex data. The body is meant to be read in batches, so files larger than 2GB are supported.
// Caller is responsible for disposing the returned stream.
public static FileStream OpenAndReadHeader(string filePath, out int vertexCount, out int vertexStride, out List<(string, ElementType)> attrs)
{
// C# arrays and NativeArrays make it hard to have a "byte" array larger than 2GB :/
if (fs.Length >= 2 * 1024 * 1024 * 1024L)
throw new IOException($"PLY {filePath} read error: currently files larger than 2GB are not supported");
var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read);
try
{
ReadHeaderImpl(filePath, out vertexCount, out vertexStride, out attrs, fs);
}
catch
{
fs.Dispose();
throw;
}
return fs;
}

static void ReadHeaderImpl(string filePath, out int vertexCount, out int vertexStride, out List<(string, ElementType)> attrs, FileStream fs)
{
// read header
vertexCount = 0;
vertexStride = 0;
Expand Down Expand Up @@ -64,17 +77,6 @@ static void ReadHeaderImpl(string filePath, out int vertexCount, out int vertexS
}
}

public static void ReadFile(string filePath, out int vertexCount, out int vertexStride, out List<(string, ElementType)> attrs, out NativeArray<byte> vertices)
{
using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read);
ReadHeaderImpl(filePath, out vertexCount, out vertexStride, out attrs, fs);

vertices = new NativeArray<byte>(vertexCount * vertexStride, Allocator.Persistent);
var readBytes = fs.Read(vertices);
if (readBytes != vertices.Length)
throw new IOException($"PLY {filePath} read error, expected {vertices.Length} data bytes got {readBytes}");
}

public enum ElementType
{
None,
Expand Down