From cd2529ee1548d5d2e800ff91b1b4a3d2df1402b7 Mon Sep 17 00:00:00 2001 From: Koray Date: Sun, 15 Feb 2026 16:03:18 +0300 Subject: [PATCH] prevent leaf render AV and harden leaf shader setup for speedtree - Guard leaf LOD access in CSpeedTreeWrapper::RenderLeaves() to avoid out-of-bounds indexing of m_pLeafVertexBuffer and m_pLeavesUpdatedByCpu. - Skip leaf update/draw when LOD is invalid, inactive, empty, or has no vertex buffer. - Add early safety returns for missing leaf buffer arrays and zero LOD count. - Guard CSpeedTreeWrapper::EndLeafForTreeType() against null m_pLeavesUpdatedByCpu. - Add shader re-init path in wrapper render entrypoints via CSpeedTreeForestDirectX8::EnsureVertexShaders() when cached shader state is missing. - Introduce public EnsureVertexShaders() in SpeedTreeForestDirectX8 while keeping InitVertexShaders() private (resolves private-access compile error). - Fix leaf shader input declaration to emit dcl_texcoord2 v9 for both GPU wind and GPU leaf placement configurations. - Make LoadLeafShader() atomic: create new shader/decl first, then swap only if both succeed, preserving previous valid shader state on failure. - Improve leaf shader failure logs to include HRESULT for easier diagnosis. --- src/SpeedTreeLib/SpeedTreeForestDirectX8.h | 1 + src/SpeedTreeLib/SpeedTreeWrapper.cpp | 87 +++++++++++++++------- src/SpeedTreeLib/VertexShaders.h | 49 ++++++++---- 3 files changed, 95 insertions(+), 42 deletions(-) diff --git a/src/SpeedTreeLib/SpeedTreeForestDirectX8.h b/src/SpeedTreeLib/SpeedTreeForestDirectX8.h index 5cfa5f0..d6c9249 100644 --- a/src/SpeedTreeLib/SpeedTreeForestDirectX8.h +++ b/src/SpeedTreeLib/SpeedTreeForestDirectX8.h @@ -52,6 +52,7 @@ class CSpeedTreeForestDirectX8 : public CSpeedTreeForest, public CGraphicBase, p void Render(unsigned long ulRenderBitVector = Forest_RenderAll); bool SetRenderingDevice(LPDIRECT3DDEVICE9 pDevice); + bool EnsureVertexShaders() { return m_pDx ? InitVertexShaders() : false; } private: bool InitVertexShaders(); diff --git a/src/SpeedTreeLib/SpeedTreeWrapper.cpp b/src/SpeedTreeLib/SpeedTreeWrapper.cpp index 4a2ff3b..b902cff 100644 --- a/src/SpeedTreeLib/SpeedTreeWrapper.cpp +++ b/src/SpeedTreeLib/SpeedTreeWrapper.cpp @@ -91,6 +91,9 @@ void CSpeedTreeWrapper::SetVertexShaders(LPDIRECT3DVERTEXDECLARATION9 pBranchVer void CSpeedTreeWrapper::OnRenderPCBlocker() { + if (!ms_dwBranchVertexShader || !ms_pLeafVertexShaderDecl || !ms_pLeafVertexShader) + CSpeedTreeForestDirectX8::Instance().EnsureVertexShaders(); + if (ms_dwBranchVertexShader == 0) { ms_dwBranchVertexShader = LoadBranchShader(ms_lpd3dDevice); @@ -200,22 +203,25 @@ void CSpeedTreeWrapper::OnRenderPCBlocker() } RenderFronds(); - STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl); - STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader); - -// SetupLeafForTreeType(); + if (ms_pLeafVertexShaderDecl && ms_pLeafVertexShader) { - // pass leaf tables to shader -#ifdef WRAPPER_USE_GPU_LEAF_PLACEMENT - UploadLeafTables(c_nVertexShader_LeafTables); -#endif + STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl); + STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader); - if (!m_CompositeImageInstance.IsEmpty()) - STATEMANAGER.SetTexture(0, m_CompositeImageInstance.GetTextureReference().GetD3DTexture()); +// SetupLeafForTreeType(); + { + // pass leaf tables to shader +#ifdef WRAPPER_USE_GPU_LEAF_PLACEMENT + UploadLeafTables(c_nVertexShader_LeafTables); +#endif + + if (!m_CompositeImageInstance.IsEmpty()) + STATEMANAGER.SetTexture(0, m_CompositeImageInstance.GetTextureReference().GetD3DTexture()); + } + RenderLeaves(); + STATEMANAGER.RestoreVertexShader(); } - RenderLeaves(); EndLeafForTreeType(); - STATEMANAGER.RestoreVertexShader(); STATEMANAGER.SetRenderState(D3DRS_LIGHTING, FALSE); STATEMANAGER.SetRenderState(D3DRS_COLORVERTEX, FALSE); @@ -234,6 +240,9 @@ void CSpeedTreeWrapper::OnRenderPCBlocker() void CSpeedTreeWrapper::OnRender() { + if (!ms_dwBranchVertexShader || !ms_pLeafVertexShaderDecl || !ms_pLeafVertexShader) + CSpeedTreeForestDirectX8::Instance().EnsureVertexShaders(); + if (ms_dwBranchVertexShader == 0) { ms_dwBranchVertexShader = LoadBranchShader(ms_lpd3dDevice); @@ -280,13 +289,16 @@ void CSpeedTreeWrapper::OnRender() SetupFrondForTreeType(); RenderFronds(); - STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl); - STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader); - - SetupLeafForTreeType(); - RenderLeaves(); + if (ms_pLeafVertexShaderDecl && ms_pLeafVertexShader) + { + STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl); + STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader); + + SetupLeafForTreeType(); + RenderLeaves(); + STATEMANAGER.RestoreVertexShader(); + } EndLeafForTreeType(); - STATEMANAGER.RestoreVertexShader(); STATEMANAGER.SetRenderState(D3DRS_LIGHTING, FALSE); STATEMANAGER.SetRenderState(D3DRS_COLORVERTEX, FALSE); @@ -1114,6 +1126,11 @@ void CSpeedTreeWrapper::RenderLeaves(void) const { // update leaf geometry m_pSpeedTree->GetGeometry(*m_pGeometryCache, SpeedTree_LeafGeometry); + + if (!m_pLeafVertexBuffer || m_usNumLeafLods == 0) + return; + + const int maxLeafLod = static_cast(m_usNumLeafLods); // update the LOD level vertex arrays we need #if defined(WRAPPER_USE_GPU_LEAF_PLACEMENT) && defined(WRAPPER_USE_GPU_WIND) @@ -1126,9 +1143,18 @@ void CSpeedTreeWrapper::RenderLeaves(void) const // reference to leaf structure const CSpeedTreeRT::SGeometry::SLeaf* pLeaf = (i == 0) ? &m_pGeometryCache->m_sLeaves0 : &m_pGeometryCache->m_sLeaves1; int unLod = pLeaf->m_nDiscreteLodLevel; + + if (!pLeaf->m_bIsActive || pLeaf->m_usLeafCount == 0) + continue; + + if (unLod < 0 || unLod >= maxLeafLod) + continue; + + if (!m_pLeafVertexBuffer[unLod]) + continue; #if defined WRAPPER_USE_GPU_LEAF_PLACEMENT - if (pLeaf->m_bIsActive && !m_pLeavesUpdatedByCpu[unLod]) + if (m_pLeavesUpdatedByCpu && !m_pLeavesUpdatedByCpu[unLod]) { // update the centers SFVFLeafVertex* pVertex = NULL; @@ -1147,7 +1173,6 @@ void CSpeedTreeWrapper::RenderLeaves(void) const m_pLeavesUpdatedByCpu[unLod] = true; } #else - if (pLeaf->m_bIsActive && m_pLeafVertexBuffer[unLod]) { // update the vertex positions SFVFLeafVertex * pVertex = NULL; @@ -1205,14 +1230,17 @@ void CSpeedTreeWrapper::RenderLeaves(void) const int unLod = pLeaf->m_nDiscreteLodLevel; - if (unLod > -1 && pLeaf->m_bIsActive && pLeaf->m_usLeafCount > 0) - { - STATEMANAGER.SetStreamSource(0, m_pLeafVertexBuffer[unLod], sizeof(SFVFLeafVertex)); - STATEMANAGER.SetRenderState(D3DRS_ALPHAREF, DWORD(pLeaf->m_fAlphaTestValue)); - - ms_faceCount += pLeaf->m_usLeafCount * 2; - STATEMANAGER.DrawPrimitive(D3DPT_TRIANGLELIST, 0, pLeaf->m_usLeafCount * 2); - } + if (unLod < 0 || unLod >= maxLeafLod || !pLeaf->m_bIsActive || pLeaf->m_usLeafCount == 0) + continue; + + if (!m_pLeafVertexBuffer[unLod]) + continue; + + STATEMANAGER.SetStreamSource(0, m_pLeafVertexBuffer[unLod], sizeof(SFVFLeafVertex)); + STATEMANAGER.SetRenderState(D3DRS_ALPHAREF, DWORD(pLeaf->m_fAlphaTestValue)); + + ms_faceCount += pLeaf->m_usLeafCount * 2; + STATEMANAGER.DrawPrimitive(D3DPT_TRIANGLELIST, 0, pLeaf->m_usLeafCount * 2); } } @@ -1222,6 +1250,9 @@ void CSpeedTreeWrapper::RenderLeaves(void) const void CSpeedTreeWrapper::EndLeafForTreeType(void) { + if (!m_pLeavesUpdatedByCpu) + return; + // reset copy flags for CPU wind for (UINT i = 0; i < m_usNumLeafLods; ++i) m_pLeavesUpdatedByCpu[i] = false; diff --git a/src/SpeedTreeLib/VertexShaders.h b/src/SpeedTreeLib/VertexShaders.h index 2670803..84531a4 100644 --- a/src/SpeedTreeLib/VertexShaders.h +++ b/src/SpeedTreeLib/VertexShaders.h @@ -204,7 +204,7 @@ static const char g_achLeafVertexProgram[] = "dcl_normal v3\n" #endif "dcl_texcoord0 v7\n" -#ifdef WRAPPER_USE_GPU_LEAF_PLACEMENT +#if defined WRAPPER_USE_GPU_WIND || defined WRAPPER_USE_GPU_LEAF_PLACEMENT "dcl_texcoord2 v9\n" #endif @@ -259,9 +259,6 @@ static const char g_achLeafVertexProgram[] = static void LoadLeafShader(LPDIRECT3DDEVICE9 pDx, LPDIRECT3DVERTEXDECLARATION9& pVertexDecl, LPDIRECT3DVERTEXSHADER9& pVertexShader) { - SAFE_RELEASE(pVertexDecl); - SAFE_RELEASE(pVertexShader); - const D3DVERTEXELEMENT9 leafVertexDecl[] = { { 0, 0, D3DDECLTYPE_FLOAT3, D3DDECLMETHOD_DEFAULT, D3DDECLUSAGE_POSITION, 0 }, #ifdef WRAPPER_USE_DYNAMIC_LIGHTING @@ -280,18 +277,42 @@ static void LoadLeafShader(LPDIRECT3DDEVICE9 pDx, LPDIRECT3DVERTEXDECLARATION9& D3DDECL_END() }; - LPD3DXBUFFER pCode = nullptr, pError = nullptr; - if (D3DXAssembleShader(g_achLeafVertexProgram, sizeof(g_achLeafVertexProgram) - 1, nullptr, nullptr, 0, &pCode, &pError) == D3D_OK) { - if (pDx->CreateVertexShader((DWORD*)pCode->GetBufferPointer(), &pVertexShader) != D3D_OK) { - TraceError("Failed to create leaf vertex shader."); - } - } - else { - TraceError("Failed to assemble leaf vertex shader. The error reported is [ %s ].", pError ? pError->GetBufferPointer() : "unknown"); + if (!pDx) + { + TraceError("Failed to load leaf shader: null D3D device."); + return; } - if (FAILED(pDx->CreateVertexDeclaration(leafVertexDecl, &pVertexDecl))) { - TraceError("Failed to create leaf vertex declaration"); + LPDIRECT3DVERTEXDECLARATION9 pNewVertexDecl = nullptr; + LPDIRECT3DVERTEXSHADER9 pNewVertexShader = nullptr; + LPD3DXBUFFER pCode = nullptr, pError = nullptr; + const HRESULT hrAssemble = D3DXAssembleShader(g_achLeafVertexProgram, sizeof(g_achLeafVertexProgram) - 1, nullptr, nullptr, 0, &pCode, &pError); + if (SUCCEEDED(hrAssemble) && pCode) + { + const HRESULT hrCreateShader = pDx->CreateVertexShader((DWORD*)pCode->GetBufferPointer(), &pNewVertexShader); + if (FAILED(hrCreateShader)) + TraceError("Failed to create leaf vertex shader (hr=0x%08X).", hrCreateShader); + } + else + { + TraceError("Failed to assemble leaf vertex shader (hr=0x%08X). The error reported is [ %s ].", hrAssemble, pError ? pError->GetBufferPointer() : "unknown"); + } + + const HRESULT hrCreateDecl = pDx->CreateVertexDeclaration(leafVertexDecl, &pNewVertexDecl); + if (FAILED(hrCreateDecl)) + TraceError("Failed to create leaf vertex declaration (hr=0x%08X).", hrCreateDecl); + + if (pNewVertexDecl && pNewVertexShader) + { + SAFE_RELEASE(pVertexDecl); + SAFE_RELEASE(pVertexShader); + pVertexDecl = pNewVertexDecl; + pVertexShader = pNewVertexShader; + } + else + { + SAFE_RELEASE(pNewVertexDecl); + SAFE_RELEASE(pNewVertexShader); } SAFE_RELEASE(pCode);