prevent leaf render AV and harden leaf shader setup for speedtree

- Guard leaf LOD access in CSpeedTreeWrapper::RenderLeaves() to avoid out-of-bounds indexing of m_pLeafVertexBuffer and m_pLeavesUpdatedByCpu.
  - Skip leaf update/draw when LOD is invalid, inactive, empty, or has no vertex buffer.
  - Add early safety returns for missing leaf buffer arrays and zero LOD count.
  - Guard CSpeedTreeWrapper::EndLeafForTreeType() against null m_pLeavesUpdatedByCpu.
  - Add shader re-init path in wrapper render entrypoints via CSpeedTreeForestDirectX8::EnsureVertexShaders() when cached shader state is missing.
  - Introduce public EnsureVertexShaders() in SpeedTreeForestDirectX8 while keeping InitVertexShaders() private (resolves private-access compile error).
  - Fix leaf shader input declaration to emit dcl_texcoord2 v9 for both GPU wind and GPU leaf placement configurations.
  - Make LoadLeafShader() atomic: create new shader/decl first, then swap only if both succeed, preserving previous valid shader state on failure.
  - Improve leaf shader failure logs to include HRESULT for easier diagnosis.
This commit is contained in:
Koray
2026-02-15 16:03:18 +03:00
parent cfe3c3cb7c
commit cd2529ee15
3 changed files with 95 additions and 42 deletions

View File

@@ -52,6 +52,7 @@ class CSpeedTreeForestDirectX8 : public CSpeedTreeForest, public CGraphicBase, p
void Render(unsigned long ulRenderBitVector = Forest_RenderAll); void Render(unsigned long ulRenderBitVector = Forest_RenderAll);
bool SetRenderingDevice(LPDIRECT3DDEVICE9 pDevice); bool SetRenderingDevice(LPDIRECT3DDEVICE9 pDevice);
bool EnsureVertexShaders() { return m_pDx ? InitVertexShaders() : false; }
private: private:
bool InitVertexShaders(); bool InitVertexShaders();

View File

@@ -91,6 +91,9 @@ void CSpeedTreeWrapper::SetVertexShaders(LPDIRECT3DVERTEXDECLARATION9 pBranchVer
void CSpeedTreeWrapper::OnRenderPCBlocker() void CSpeedTreeWrapper::OnRenderPCBlocker()
{ {
if (!ms_dwBranchVertexShader || !ms_pLeafVertexShaderDecl || !ms_pLeafVertexShader)
CSpeedTreeForestDirectX8::Instance().EnsureVertexShaders();
if (ms_dwBranchVertexShader == 0) if (ms_dwBranchVertexShader == 0)
{ {
ms_dwBranchVertexShader = LoadBranchShader(ms_lpd3dDevice); ms_dwBranchVertexShader = LoadBranchShader(ms_lpd3dDevice);
@@ -200,22 +203,25 @@ void CSpeedTreeWrapper::OnRenderPCBlocker()
} }
RenderFronds(); RenderFronds();
STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl); if (ms_pLeafVertexShaderDecl && ms_pLeafVertexShader)
STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader); {
STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl);
STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader);
// SetupLeafForTreeType(); // SetupLeafForTreeType();
{ {
// pass leaf tables to shader // pass leaf tables to shader
#ifdef WRAPPER_USE_GPU_LEAF_PLACEMENT #ifdef WRAPPER_USE_GPU_LEAF_PLACEMENT
UploadLeafTables(c_nVertexShader_LeafTables); UploadLeafTables(c_nVertexShader_LeafTables);
#endif #endif
if (!m_CompositeImageInstance.IsEmpty()) if (!m_CompositeImageInstance.IsEmpty())
STATEMANAGER.SetTexture(0, m_CompositeImageInstance.GetTextureReference().GetD3DTexture()); STATEMANAGER.SetTexture(0, m_CompositeImageInstance.GetTextureReference().GetD3DTexture());
}
RenderLeaves();
STATEMANAGER.RestoreVertexShader();
} }
RenderLeaves();
EndLeafForTreeType(); EndLeafForTreeType();
STATEMANAGER.RestoreVertexShader();
STATEMANAGER.SetRenderState(D3DRS_LIGHTING, FALSE); STATEMANAGER.SetRenderState(D3DRS_LIGHTING, FALSE);
STATEMANAGER.SetRenderState(D3DRS_COLORVERTEX, FALSE); STATEMANAGER.SetRenderState(D3DRS_COLORVERTEX, FALSE);
@@ -234,6 +240,9 @@ void CSpeedTreeWrapper::OnRenderPCBlocker()
void CSpeedTreeWrapper::OnRender() void CSpeedTreeWrapper::OnRender()
{ {
if (!ms_dwBranchVertexShader || !ms_pLeafVertexShaderDecl || !ms_pLeafVertexShader)
CSpeedTreeForestDirectX8::Instance().EnsureVertexShaders();
if (ms_dwBranchVertexShader == 0) if (ms_dwBranchVertexShader == 0)
{ {
ms_dwBranchVertexShader = LoadBranchShader(ms_lpd3dDevice); ms_dwBranchVertexShader = LoadBranchShader(ms_lpd3dDevice);
@@ -280,13 +289,16 @@ void CSpeedTreeWrapper::OnRender()
SetupFrondForTreeType(); SetupFrondForTreeType();
RenderFronds(); RenderFronds();
STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl); if (ms_pLeafVertexShaderDecl && ms_pLeafVertexShader)
STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader); {
STATEMANAGER.SetVertexDeclaration(ms_pLeafVertexShaderDecl);
STATEMANAGER.SaveVertexShader(ms_pLeafVertexShader);
SetupLeafForTreeType(); SetupLeafForTreeType();
RenderLeaves(); RenderLeaves();
STATEMANAGER.RestoreVertexShader();
}
EndLeafForTreeType(); EndLeafForTreeType();
STATEMANAGER.RestoreVertexShader();
STATEMANAGER.SetRenderState(D3DRS_LIGHTING, FALSE); STATEMANAGER.SetRenderState(D3DRS_LIGHTING, FALSE);
STATEMANAGER.SetRenderState(D3DRS_COLORVERTEX, FALSE); STATEMANAGER.SetRenderState(D3DRS_COLORVERTEX, FALSE);
@@ -1115,6 +1127,11 @@ void CSpeedTreeWrapper::RenderLeaves(void) const
// update leaf geometry // update leaf geometry
m_pSpeedTree->GetGeometry(*m_pGeometryCache, SpeedTree_LeafGeometry); m_pSpeedTree->GetGeometry(*m_pGeometryCache, SpeedTree_LeafGeometry);
if (!m_pLeafVertexBuffer || m_usNumLeafLods == 0)
return;
const int maxLeafLod = static_cast<int>(m_usNumLeafLods);
// update the LOD level vertex arrays we need // update the LOD level vertex arrays we need
#if defined(WRAPPER_USE_GPU_LEAF_PLACEMENT) && defined(WRAPPER_USE_GPU_WIND) #if defined(WRAPPER_USE_GPU_LEAF_PLACEMENT) && defined(WRAPPER_USE_GPU_WIND)
// do nothing, needs no updates // do nothing, needs no updates
@@ -1127,8 +1144,17 @@ void CSpeedTreeWrapper::RenderLeaves(void) const
const CSpeedTreeRT::SGeometry::SLeaf* pLeaf = (i == 0) ? &m_pGeometryCache->m_sLeaves0 : &m_pGeometryCache->m_sLeaves1; const CSpeedTreeRT::SGeometry::SLeaf* pLeaf = (i == 0) ? &m_pGeometryCache->m_sLeaves0 : &m_pGeometryCache->m_sLeaves1;
int unLod = pLeaf->m_nDiscreteLodLevel; int unLod = pLeaf->m_nDiscreteLodLevel;
if (!pLeaf->m_bIsActive || pLeaf->m_usLeafCount == 0)
continue;
if (unLod < 0 || unLod >= maxLeafLod)
continue;
if (!m_pLeafVertexBuffer[unLod])
continue;
#if defined WRAPPER_USE_GPU_LEAF_PLACEMENT #if defined WRAPPER_USE_GPU_LEAF_PLACEMENT
if (pLeaf->m_bIsActive && !m_pLeavesUpdatedByCpu[unLod]) if (m_pLeavesUpdatedByCpu && !m_pLeavesUpdatedByCpu[unLod])
{ {
// update the centers // update the centers
SFVFLeafVertex* pVertex = NULL; SFVFLeafVertex* pVertex = NULL;
@@ -1147,7 +1173,6 @@ void CSpeedTreeWrapper::RenderLeaves(void) const
m_pLeavesUpdatedByCpu[unLod] = true; m_pLeavesUpdatedByCpu[unLod] = true;
} }
#else #else
if (pLeaf->m_bIsActive && m_pLeafVertexBuffer[unLod])
{ {
// update the vertex positions // update the vertex positions
SFVFLeafVertex * pVertex = NULL; SFVFLeafVertex * pVertex = NULL;
@@ -1205,14 +1230,17 @@ void CSpeedTreeWrapper::RenderLeaves(void) const
int unLod = pLeaf->m_nDiscreteLodLevel; int unLod = pLeaf->m_nDiscreteLodLevel;
if (unLod > -1 && pLeaf->m_bIsActive && pLeaf->m_usLeafCount > 0) if (unLod < 0 || unLod >= maxLeafLod || !pLeaf->m_bIsActive || pLeaf->m_usLeafCount == 0)
{ continue;
STATEMANAGER.SetStreamSource(0, m_pLeafVertexBuffer[unLod], sizeof(SFVFLeafVertex));
STATEMANAGER.SetRenderState(D3DRS_ALPHAREF, DWORD(pLeaf->m_fAlphaTestValue));
ms_faceCount += pLeaf->m_usLeafCount * 2; if (!m_pLeafVertexBuffer[unLod])
STATEMANAGER.DrawPrimitive(D3DPT_TRIANGLELIST, 0, pLeaf->m_usLeafCount * 2); continue;
}
STATEMANAGER.SetStreamSource(0, m_pLeafVertexBuffer[unLod], sizeof(SFVFLeafVertex));
STATEMANAGER.SetRenderState(D3DRS_ALPHAREF, DWORD(pLeaf->m_fAlphaTestValue));
ms_faceCount += pLeaf->m_usLeafCount * 2;
STATEMANAGER.DrawPrimitive(D3DPT_TRIANGLELIST, 0, pLeaf->m_usLeafCount * 2);
} }
} }
@@ -1222,6 +1250,9 @@ void CSpeedTreeWrapper::RenderLeaves(void) const
void CSpeedTreeWrapper::EndLeafForTreeType(void) void CSpeedTreeWrapper::EndLeafForTreeType(void)
{ {
if (!m_pLeavesUpdatedByCpu)
return;
// reset copy flags for CPU wind // reset copy flags for CPU wind
for (UINT i = 0; i < m_usNumLeafLods; ++i) for (UINT i = 0; i < m_usNumLeafLods; ++i)
m_pLeavesUpdatedByCpu[i] = false; m_pLeavesUpdatedByCpu[i] = false;

View File

@@ -204,7 +204,7 @@ static const char g_achLeafVertexProgram[] =
"dcl_normal v3\n" "dcl_normal v3\n"
#endif #endif
"dcl_texcoord0 v7\n" "dcl_texcoord0 v7\n"
#ifdef WRAPPER_USE_GPU_LEAF_PLACEMENT #if defined WRAPPER_USE_GPU_WIND || defined WRAPPER_USE_GPU_LEAF_PLACEMENT
"dcl_texcoord2 v9\n" "dcl_texcoord2 v9\n"
#endif #endif
@@ -259,9 +259,6 @@ static const char g_achLeafVertexProgram[] =
static void LoadLeafShader(LPDIRECT3DDEVICE9 pDx, LPDIRECT3DVERTEXDECLARATION9& pVertexDecl, LPDIRECT3DVERTEXSHADER9& pVertexShader) static void LoadLeafShader(LPDIRECT3DDEVICE9 pDx, LPDIRECT3DVERTEXDECLARATION9& pVertexDecl, LPDIRECT3DVERTEXSHADER9& pVertexShader)
{ {
SAFE_RELEASE(pVertexDecl);
SAFE_RELEASE(pVertexShader);
const D3DVERTEXELEMENT9 leafVertexDecl[] = { const D3DVERTEXELEMENT9 leafVertexDecl[] = {
{ 0, 0, D3DDECLTYPE_FLOAT3, D3DDECLMETHOD_DEFAULT, D3DDECLUSAGE_POSITION, 0 }, { 0, 0, D3DDECLTYPE_FLOAT3, D3DDECLMETHOD_DEFAULT, D3DDECLUSAGE_POSITION, 0 },
#ifdef WRAPPER_USE_DYNAMIC_LIGHTING #ifdef WRAPPER_USE_DYNAMIC_LIGHTING
@@ -280,18 +277,42 @@ static void LoadLeafShader(LPDIRECT3DDEVICE9 pDx, LPDIRECT3DVERTEXDECLARATION9&
D3DDECL_END() D3DDECL_END()
}; };
LPD3DXBUFFER pCode = nullptr, pError = nullptr; if (!pDx)
if (D3DXAssembleShader(g_achLeafVertexProgram, sizeof(g_achLeafVertexProgram) - 1, nullptr, nullptr, 0, &pCode, &pError) == D3D_OK) { {
if (pDx->CreateVertexShader((DWORD*)pCode->GetBufferPointer(), &pVertexShader) != D3D_OK) { TraceError("Failed to load leaf shader: null D3D device.");
TraceError("Failed to create leaf vertex shader."); return;
}
}
else {
TraceError("Failed to assemble leaf vertex shader. The error reported is [ %s ].", pError ? pError->GetBufferPointer() : "unknown");
} }
if (FAILED(pDx->CreateVertexDeclaration(leafVertexDecl, &pVertexDecl))) { LPDIRECT3DVERTEXDECLARATION9 pNewVertexDecl = nullptr;
TraceError("Failed to create leaf vertex declaration"); LPDIRECT3DVERTEXSHADER9 pNewVertexShader = nullptr;
LPD3DXBUFFER pCode = nullptr, pError = nullptr;
const HRESULT hrAssemble = D3DXAssembleShader(g_achLeafVertexProgram, sizeof(g_achLeafVertexProgram) - 1, nullptr, nullptr, 0, &pCode, &pError);
if (SUCCEEDED(hrAssemble) && pCode)
{
const HRESULT hrCreateShader = pDx->CreateVertexShader((DWORD*)pCode->GetBufferPointer(), &pNewVertexShader);
if (FAILED(hrCreateShader))
TraceError("Failed to create leaf vertex shader (hr=0x%08X).", hrCreateShader);
}
else
{
TraceError("Failed to assemble leaf vertex shader (hr=0x%08X). The error reported is [ %s ].", hrAssemble, pError ? pError->GetBufferPointer() : "unknown");
}
const HRESULT hrCreateDecl = pDx->CreateVertexDeclaration(leafVertexDecl, &pNewVertexDecl);
if (FAILED(hrCreateDecl))
TraceError("Failed to create leaf vertex declaration (hr=0x%08X).", hrCreateDecl);
if (pNewVertexDecl && pNewVertexShader)
{
SAFE_RELEASE(pVertexDecl);
SAFE_RELEASE(pVertexShader);
pVertexDecl = pNewVertexDecl;
pVertexShader = pNewVertexShader;
}
else
{
SAFE_RELEASE(pNewVertexDecl);
SAFE_RELEASE(pNewVertexShader);
} }
SAFE_RELEASE(pCode); SAFE_RELEASE(pCode);