Visitar URL original
Add SOG v3 loading and over-unity opacity rendering by slimbuck · Pull Request #9873 · playcanvas/engine · GitHub
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/framework/parsers/sog-bundle.js
Original file line number Diff line number Diff line change
Expand Up @@ -258,8 +258,13 @@ class SogBundleParser {
return;
}

// extract filenames from meta.json
const filenames = ['means', 'scales', 'quats', 'sh0', 'shN'].map(key => meta[key]?.files ?? []).flat();
if (meta.version !== undefined && meta.version !== 2 && meta.version !== 3) {
callback(`Error loading gsplat: unsupported SOG version ${meta.version} in ${url.original}`);
return;
}

// extract filenames from meta.json (v3 has no quats and may carry blind data)
const filenames = ['means', 'scales', 'quats', 'sh0', 'shN', 'blind'].map(key => meta[key]?.files ?? []).flat();

const promises = [];
for (const filename of filenames) {
Expand Down Expand Up @@ -333,11 +338,12 @@ class SogBundleParser {
data.numSplats = meta.count;
data.means_l = textures[meta.means.files[0]].resource;
data.means_u = textures[meta.means.files[1]].resource;
data.quats = textures[meta.quats.files[0]].resource;
data.quats = meta.quats ? textures[meta.quats.files[0]].resource : null;
data.scales = textures[meta.scales.files[0]].resource;
data.sh0 = textures[meta.sh0.files[0]].resource;
data.sh_centroids = textures[meta.shN?.files[0]]?.resource;
data.sh_labels = textures[meta.shN?.files[1]]?.resource;
data.blind = meta.blind ? textures[meta.blind.files[0]]?.resource ?? null : null;
data.shBands = GSplatSogData.calcBands(data.sh_centroids?.width);

if (!decompress) {
Expand Down
12 changes: 9 additions & 3 deletions src/framework/parsers/sog.js
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,19 @@ class SogParser {
const gsplatCentersEnabledAtLoad = this.app.scene?.gsplatCentersEnabled !== false;

// transform meta to latest shape
if (meta.version !== 2) {
if (meta.version === undefined) {
Debug.deprecated('Loading SOG v1 data which is deprecated. Please recompress your scene with latest tools.');
meta = upgradeMeta(meta);
} else if (meta.version !== 2 && meta.version !== 3) {
callback(`Error loading gsplat: unsupported SOG version ${meta.version} in ${url.original}`);
return;
}

const { assets } = this.app;

const subs = ['means', 'quats', 'scales', 'sh0', 'shN'];
// v3 has no quats texture (the rotation lives in the alpha channels of means_l, means_u
// and scales) and may carry per-gaussian blind data
const subs = ['means', 'quats', 'scales', 'sh0', 'shN', 'blind'];

const textures = {};
const promises = [];
Expand Down Expand Up @@ -222,11 +227,12 @@ class SogParser {
data.numSplats = meta.count;
data.means_l = textures.means[0].resource;
data.means_u = textures.means[1].resource;
data.quats = textures.quats[0].resource;
data.quats = textures.quats[0]?.resource ?? null;
data.scales = textures.scales[0].resource;
data.sh0 = textures.sh0[0].resource;
data.sh_centroids = textures.shN?.[0]?.resource;
data.sh_labels = textures.shN?.[1]?.resource;
data.blind = textures.blind?.[0]?.resource ?? null;
data.shBands = GSplatSogData.calcBands(data.sh_centroids?.width);

const decompress = asset.data?.decompress;
Expand Down
26 changes: 22 additions & 4 deletions src/scene/gsplat-unified/gsplat-hybrid-renderer.js
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ class GSplatHybridRenderer extends GSplatRenderer {
this._internalDefines.add('GSPLAT_UNIFIED_ID');
this._internalDefines.add('PICK_CUSTOM_ID');
this._internalDefines.add('GSPLAT_OVERDRAW');
this._internalDefines.add('GSPLAT_OVERUNITY');
this._internalDefines.add('GSPLAT_NO_FOG');
this._internalDefines.add('GSPLAT_NO_TONEMAP');
this._internalDefines.add('GSPLAT_XR');
Expand Down Expand Up @@ -546,7 +547,7 @@ class GSplatHybridRenderer extends GSplatRenderer {
antiAlias: params.antiAlias,
isStereo,
material: params.material,
userCacheWords: params.varyings.words
userCacheWords: params.varyings.words + this._plateauWords()
});

projector.writeIndirectArgs(
Expand Down Expand Up @@ -745,6 +746,7 @@ class GSplatHybridRenderer extends GSplatRenderer {

this._pickMaterial.setDefine('{GSPLAT_INSTANCE_SIZE}', GSplatResourceBase.instanceSize);
this._pickMaterial.setDefine('{CACHE_STRIDE}', this._cacheStride);
this._pickMaterial.setDefine('GSPLAT_OVERUNITY', !!this._material.getDefine('GSPLAT_OVERUNITY'));
this._pickMaterial.setDefine('SH_BANDS', '0');
this._pickMaterial.setDefine('GSPLAT_INDIRECT_DRAW', true);
this._pickMaterial.setDefine('DITHER_NONE', '');
Expand Down Expand Up @@ -860,14 +862,19 @@ class GSplatHybridRenderer extends GSplatRenderer {
this._material.update();
}

// keep the projection cache stride in sync with the user varying streams
const cacheStride = CACHE_STRIDE + params.varyings.words;
if (cacheStride !== this._cacheStride) {
// keep the projection cache stride in sync with the user varying streams and the
// over-unity plateau word
const plateauWords = this._plateauWords();
const cacheStride = CACHE_STRIDE + params.varyings.words + plateauWords;
const overUnity = plateauWords > 0;
if (cacheStride !== this._cacheStride || overUnity !== !!this._material.getDefine('GSPLAT_OVERUNITY')) {
this._cacheStride = cacheStride;
this._material.setDefine('{CACHE_STRIDE}', cacheStride);
this._material.setDefine('GSPLAT_OVERUNITY', overUnity);
this._material.update();
if (this._pickMaterial) {
this._pickMaterial.setDefine('{CACHE_STRIDE}', cacheStride);
this._pickMaterial.setDefine('GSPLAT_OVERUNITY', overUnity);
this._pickMaterial.update();
}
}
Expand Down Expand Up @@ -932,6 +939,17 @@ class GSplatHybridRenderer extends GSplatRenderer {
this._material.update();
}

/**
* Cache words for the over-unity plateau shape: one when the work buffer carries it (see
* GSplatParams#compensation).
*
* @returns {number} The word count.
* @private
*/
_plateauWords() {
return this.workBuffer?.format.getStream('splatPlateau') ? 1 : 0;
}

/**
* Updates pick ID defines to match the work-buffer format.
*
Expand Down
62 changes: 55 additions & 7 deletions src/scene/gsplat-unified/gsplat-manager.js
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,14 @@ class GSplatManager {
/** @type {GSplatParams} */
gsplat;

/**
* World state version and compensation setting last checked for over-unity resources.
*
* @type {string}
* @private
*/
_overUnityCheckKey = '';

/** @type {GSplatRenderer} */
renderer;

Expand Down Expand Up @@ -626,12 +634,7 @@ class GSplatManager {

// Detect work-buffer format changes (wholesale swap + extra streams) before world.update,
// which reads the work buffer.
this.world.syncFormat(this._formatResult);
if (this._formatResult.bufferRecreated) {
this.renderer.setDataSource(this.world.workBuffer);
this.shadowRenderer?.setDataSource(this.world.workBuffer);
}
if (this._formatResult.sortNeeded) this.sortNeeded = true;
let bufferRecreated = this._syncFormat();

// Check for runtime renderer mode changes and transition if needed.
this.prepareRendererMode();
Expand All @@ -642,17 +645,62 @@ class GSplatManager {

// World LOD / streaming / world-state creation.
this.world.update(this.cameraNode, allowLodUpdate, !!this.cpuSorter, this._updateResult);

// a resource needing the plateau stream just arrived: attach the stream before this
// frame's bake, so the bake writes it
if (this._checkOverUnity()) {
bufferRecreated = this._syncFormat() || bufferRecreated;
}
if (this._updateResult.overdrawDirty) this.renderer.updateOverdrawMode(this.gsplat);
if (this._updateResult.sortNeeded) this.sortNeeded = true;
if (this._updateResult.newVersion) this._feedCpuSorterCenters();

// tick cooldowns once per frame per unique octree, across the worlds of all managers
this.world.tickCooldowns(token);

this._streamAdvanced = this._updateResult.newVersion || this._formatResult.bufferRecreated;
this._streamAdvanced = this._updateResult.newVersion || bufferRecreated;
return this._streamAdvanced;
}

/**
* Syncs the work buffer with the format and hands a recreated buffer to the renderers.
*
* @returns {boolean} True if the work buffer was recreated.
* @private
*/
_syncFormat() {
this.world.syncFormat(this._formatResult);
if (this._formatResult.bufferRecreated) {
this.renderer.setDataSource(this.world.workBuffer);
this.shadowRenderer?.setDataSource(this.world.workBuffer);
}
if (this._formatResult.sortNeeded) this.sortNeeded = true;
return this._formatResult.bufferRecreated;
}

/**
* Enables the over-unity work buffer stream once a resource that needs it is in the world.
*
* @returns {boolean} True if the stream was enabled.
* @private
*/
_checkOverUnity() {
const gsplat = this.gsplat;
if (!gsplat.compensation || gsplat.format.getStream('splatPlateau')) return false;

// only re-scan when the world's splats or the compensation setting changed
const key = `${this.world.lastWorldStateVersion}:${gsplat._compensationVersion}`;
if (key === this._overUnityCheckKey) return false;
this._overUnityCheckKey = key;

const state = this.world.getState(this.world.lastWorldStateVersion);
if (state?.splats.some(splat => splat.resource.hasOverUnity)) {
gsplat._enableOverUnity(true);
return true;
}
return false;
}

update() {

// Reset per-frame buffer-copy stats before any bake. Must precede applyPendingSorted (whose
Expand Down
11 changes: 11 additions & 0 deletions src/scene/gsplat-unified/gsplat-octree.resource.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,17 @@ class GSplatOctreeResource {
this.data.tree = null;
}

/**
* Application-defined data stored with the streamed splat scene, as in glTF's `extras`: the
* `extras` value of its lod-meta.json. The engine attaches no meaning to it. Undefined when
* the manifest has none.
*
* @type {*}
*/
get extras() {
return this.data?.extras;
}

/**
* Total number of splats across all leaf nodes at the highest LOD (LOD 0) — the full-detail
* splat count of the captured scene. Not all of these are resident at runtime: the LOD
Expand Down
61 changes: 60 additions & 1 deletion src/scene/gsplat-unified/gsplat-params.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Debug } from '../../core/debug.js';
import {
PIXELFORMAT_R32U, PIXELFORMAT_RGBA16F, PIXELFORMAT_RGBA16U,
PIXELFORMAT_R8, PIXELFORMAT_R32U, PIXELFORMAT_RGBA16F, PIXELFORMAT_RGBA16U,
PIXELFORMAT_RGBA32U, PIXELFORMAT_RG32U
} from '../../platform/graphics/constants.js';
import { ShaderMaterial } from '../materials/shader-material.js';
Expand Down Expand Up @@ -415,6 +415,65 @@ class GSplatParams {
return this._enableIds;
}

/** @private */
_compensation = true;

/**
* Incremented whenever {@link GSplatParams#compensation} changes, so managers re-check
* whether their resources need the over-unity stream.
*
* @ignore
*/
_compensationVersion = 0;

/**
* Whether to draw over-unity opacity. Decimated splat data (SOG v3) can carry gaussians whose
* true opacity exceeds 1 - the mass of several merged gaussians that an opacity of at most 1
* cannot hold. When enabled, these are drawn with a flat-topped (plateau) profile that carries
* that mass: an opaque core joined smoothly to a gaussian edge, with a correspondingly larger
* footprint. When disabled, their opacity is clamped to 1. The extra work buffer stream this
* needs (1 byte per splat) is only allocated once such data is rendered. Defaults to true.
*
* Intended mainly for debugging and comparison.
*
* @type {boolean}
*/
set compensation(value) {
if (this._compensation !== value) {
this._compensation = value;
this._compensationVersion++;
if (!value) {
this._enableOverUnity(false);
}
}
}

/**
* Gets whether over-unity opacity is drawn.
*
* @type {boolean}
*/
get compensation() {
return this._compensation;
}

/**
* Adds or removes the work buffer stream carrying the per-splat plateau shape of over-unity
* opacity (see {@link GSplatParams#compensation}). The stream is R8 and holds log2(D') / 6,
* so 0 is an ordinary gaussian and the plateau shape D' ranges up to 64.
*
* @param {boolean} enable - Whether the stream should exist.
* @ignore
*/
_enableOverUnity(enable) {
const has = !!this._format.getStream('splatPlateau');
if (enable && !has && this._compensation) {
this._format.addExtraStreams([{ name: 'splatPlateau', format: PIXELFORMAT_R8 }]);
} else if (!enable && has) {
this._format.removeExtraStreams(['splatPlateau']);
}
}

/**
* Distance threshold in world units to trigger LOD updates for camera and gsplat instances.
* Defaults to 1.
Expand Down
5 changes: 5 additions & 0 deletions src/scene/gsplat-unified/gsplat-projector.js
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,11 @@ class GSplatProjector {
if (stereo) {
cdefines.set('GSPLAT_XR', '');
}
// over-unity plateau shape, appended as the cache's last word (the caller counts it in
// userCacheWords; see GSplatHybridRenderer)
if (wbFormat.getStream('splatPlateau')) {
cdefines.set('GSPLAT_OVERUNITY', '');
}

// Format-specific defines.
const colorStream = wbFormat.getStream('dataColor');
Expand Down
4 changes: 4 additions & 0 deletions src/scene/gsplat-unified/gsplat-quad-renderer.js
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ class GSplatQuadRenderer extends GSplatRenderer {
this._internalDefines.add('GSPLAT_NO_FOG');
this._internalDefines.add('GSPLAT_NO_TONEMAP');
this._internalDefines.add('GSPLAT_OVERDRAW');
this._internalDefines.add('GSPLAT_OVERUNITY');

this.meshInstance = this.createMeshInstance();
}
Expand Down Expand Up @@ -349,6 +350,9 @@ class GSplatQuadRenderer extends GSplatRenderer {
const hasPcId = !!this.workBuffer.format.getStream('pcId');
this._material.setDefine('GSPLAT_UNIFIED_ID', hasPcId);
this._material.setDefine('PICK_CUSTOM_ID', hasPcId);

// GSPLAT_OVERUNITY draws the over-unity plateau profile (see GSplatParams#compensation)
this._material.setDefine('GSPLAT_OVERUNITY', !!this.workBuffer.format.getStream('splatPlateau'));
}

/**
Expand Down
16 changes: 16 additions & 0 deletions src/scene/gsplat-unified/gsplat-shadow-renderer.js
Original file line number Diff line number Diff line change
Expand Up @@ -249,9 +249,19 @@ class GSplatShadowRenderer {
const wbFormat = this.world.workBuffer.format;
const version = wbFormat.extraStreamsVersion;
if (!this._cullShader || version !== this._cullFormatVersion || this._userChunksKey !== this._cullBuiltChunksKey) {
const formatChanged = version !== this._cullFormatVersion;
this._cullFormatVersion = version;
this._cullBuiltChunksKey = this._userChunksKey;
this._buildCullShader();

// streams added to the format in place (e.g. the over-unity plateau) also change the
// draw materials' read code and bindings
if (formatChanged) {
this.entries.forEach((entry) => {
this._configureMaterialWorkBuffer(entry.material);
entry.material.update();
});
}
}
}

Expand Down Expand Up @@ -307,6 +317,9 @@ class GSplatShadowRenderer {
if (colorStream && colorStream.format !== PIXELFORMAT_RGBA16U) {
cdefines.set('GSPLAT_COLOR_FLOAT', '');
}
if (wbFormat.getStream('splatPlateau')) {
cdefines.set('GSPLAT_OVERUNITY', '');
}

this._cullShader?.destroy();
this._cullShader = new Shader(device, {
Expand Down Expand Up @@ -814,6 +827,9 @@ class GSplatShadowRenderer {
material.setDefine('GSPLAT_UNIFIED_ID', hasPcId);
material.setDefine('PICK_CUSTOM_ID', hasPcId);

// over-unity plateau profile (see GSplatParams#compensation)
material.setDefine('GSPLAT_OVERUNITY', !!wbFormat.getStream('splatPlateau'));

// bind work buffer textures
for (const stream of wbFormat.resourceStreams) {
const texture = workBuffer.getTexture(stream.name);
Expand Down
Loading
Loading