refactor rendering logic to be a lot less wasteful

Observations:
- Entity::renderPass was never set to RENDER_ALL -> can simplify some things
- The initial pass check in RenderObject::render() was constant for each pass
-> All logic that is per-pass-constant can be moved to a renderability pre-check
- Core::overrideStartLayer, Core::overrideEndLayer, Core::rlayer were never used
- Should be possible eventually to prepare & render layers in parallel

I am not sure if the changes in this commit are 100% correct, but layer passes
are still working and the hug looks like it should.

Thinking about it, the overrideRenderPass functionality should never have existed.
The game scripts don't actually use entity_setRenderPass
(which in turn calls Entity::setOverrideRenderPass())
so I might remove that function in a future commit,
together with the rest of the "override" functionality.

BBGE/Core.cpp

@@ -364,13 +364,10 @@ Core::Core(const std::string &filesystem, const std::string& extraDataDir, int n
 	lib_graphics = lib_sound = lib_input = false;
 	mouseConstraint = false;
 	mouseCircle = 0;
-	overrideStartLayer = 0;
-	overrideEndLayer = 0;
 	frameOutputMode = false;
 	updateMouse = true;
 	particlesPaused = false;
 	joystickAsMouse = false;
-	currentLayerPass = 0;
 	flipMouseButtons = 0;
 	joystickEnabled = false;
 	doScreenshot = false;
@@ -382,6 +379,8 @@ Core::Core(const std::string &filesystem, const std::string& extraDataDir, int n
 	invGlobalScale = 1.0;
 	invGlobalScaleSqr = 1.0;
 	renderObjectCount = 0;
+	processedRenderObjectCount = 0;
+	totalRenderObjectCount = 0;
 	avgFPS.resize(1);
 	minimized = false;
 	shuttingDown = false;
@@ -1737,11 +1736,9 @@ void Core::updateCullData()
 
 void Core::render(int startLayer, int endLayer, bool useFrameBufferIfAvail)
 {
-	if (startLayer == -1 && endLayer == -1 && overrideStartLayer != 0)
-	{
-		startLayer = overrideStartLayer;
-		endLayer = overrideEndLayer;
-	}
+	renderObjectCount = 0;
+	processedRenderObjectCount = 0;
+	totalRenderObjectCount = 0;
 
 	globalScaleChanged();
 
@@ -1752,14 +1749,12 @@ void Core::render(int startLayer, int endLayer, bool useFrameBufferIfAvail)
 
 	updateCullData();
 
-
-
-	renderObjectCount = 0;
-	processedRenderObjectCount = 0;
-	totalRenderObjectCount = 0;
-
-	CombinedRenderAndGPUState rgstate;
-
+	// TODO: this could be done in parallel
+	for (size_t i = 0; i < renderObjectLayers.size(); ++i)
+	{
+		if(renderObjectLayers[i].visible)
+			renderObjectLayers[i].prepareRender();
+	}
 
 	glBindTexture(GL_TEXTURE_2D, 0);
 	glLoadIdentity();									// Reset The View
@@ -1773,11 +1768,7 @@ void Core::render(int startLayer, int endLayer, bool useFrameBufferIfAvail)
 	setupRenderPositionAndScale();
 
 
-
-	RenderObject::rlayer = 0;
-
 	for (size_t c = 0; c < renderObjectLayerOrder.size(); c++)
-
 	{
 		int i = renderObjectLayerOrder[c];
 		if (i == -1) continue;
@@ -1803,24 +1794,11 @@ void Core::render(int startLayer, int endLayer, bool useFrameBufferIfAvail)
 		}
 
 		RenderObjectLayer *r = &renderObjectLayers[i];
-		RenderObject::rlayer = r;
-		if (r->visible)
-		{
-			if (r->startPass == r->endPass)
-			{
-				r->renderPass(rgstate, RenderObject::RENDER_ALL);
-			}
-			else
-			{
-				for (int pass = r->startPass; pass <= r->endPass; pass++)
-				{
-					r->renderPass(rgstate, pass);
-				}
-			}
-		}
+		if(!r->visible)
+			continue;
+
+		r->render();
 	}
-
-
 }
 
 void Core::showBuffer()

BBGE/Core.h

@@ -118,9 +118,12 @@ public:
 	void remove(RenderObject* r);
 	void moveToFront(RenderObject *r);
 	void moveToBack(RenderObject *r);
-	void renderPass(const RenderState& rs, int pass);
+
 	void reloadDevice();
 
+	void prepareRender();
+	void render() const;
+
 	inline bool empty()
 	{
 		return objectCount == 0;
@@ -164,9 +167,8 @@ public:
 	bool update;
 
 protected:
-	inline void renderOneObject(const RenderState& rs, const RenderObject *robj);
-
 	RenderObjects renderObjects;
+	std::vector<const RenderObject*> toRender;
 	size_t objectCount;
 	size_t firstFreeIdx;
 	size_t iter;
@@ -337,7 +339,7 @@ public:
 	float cullRadius;
 	float cullRadiusSqr;
 	Vector cullCenter;
-	unsigned int renderObjectCount, processedRenderObjectCount, totalRenderObjectCount;
+	size_t renderObjectCount, processedRenderObjectCount, totalRenderObjectCount;
 	float invGlobalScale, invGlobalScaleSqr;
 
 	void globalScaleChanged();
@@ -349,7 +351,6 @@ public:
 	bool getKeyState(int k);
 	bool getMouseButtonState(int m);
 
-	int currentLayerPass;
 	int keys[KEY_MAXARRAY];
 	virtual void debugLog(const std::string &s);
 	virtual void errorLog(const std::string &s);
@@ -384,8 +385,6 @@ public:
 	bool updateMouse;
 	bool frameOutputMode;
 
-	int overrideStartLayer, overrideEndLayer;
-
 	ParticleEffect* createParticleEffect(const std::string &name, const Vector &position, int layer, float rotz=0);
 
 	std::string secondaryTexturePath;

BBGE/RenderObject.cpp

@@ -36,8 +36,6 @@ size_t	RenderObject::lastTextureApplied			= 0;
 bool	RenderObject::lastTextureRepeat				= false;
 bool	RenderObject::renderPaths					= false;
 
-RenderObjectLayer *RenderObject::rlayer				= 0;
-
 void RenderObject::toggleAlpha(float t)
 {
 	if (alpha.x < 0.5f)
@@ -426,6 +424,37 @@ bool RenderObject::hasRenderPass(const int pass) const
 	return false;
 }
 
+bool RenderObject::shouldTryToRender() const
+{
+	return !parent
+		&& alpha.x > 0
+		&& (!cull || isOnScreen());
+}
+
+bool RenderObject::isVisibleInPass(int pass) const
+{
+	assert(!parent); // This check should be done for root objects only
+	assert(pass != RENDER_ALL); // why call this when we already know we don't do passes
+
+	if (this->overrideRenderPass != OVERRIDE_NONE)
+	{
+		// FIXME: overrideRenderPass is not applied to the
+		// node itself in the original check (below); is
+		// that intentional?  Doing the same thing here
+		// for the time being.  --achurch
+		if (pass != this->renderPass
+			&& pass != this->overrideRenderPass)
+			return false;
+	}
+	else
+	{
+		if (!hasRenderPass(pass))
+			return false;
+	}
+
+	return true;
+}
+
 void RenderObject::render(const RenderState& rs) const
 {
 	if (isHidden()) return;
@@ -433,31 +462,6 @@ void RenderObject::render(const RenderState& rs) const
 	/// new (breaks anything?)
 	if (alpha.x == 0 || alphaMod == 0) return;
 
-	if (core->currentLayerPass != RENDER_ALL && renderPass != RENDER_ALL)
-	{
-		RenderObject *top = getTopParent();
-		if (top == NULL && this->overrideRenderPass != OVERRIDE_NONE)
-		{
-			// FIXME: overrideRenderPass is not applied to the
-			// node itself in the original check (below); is
-			// that intentional?  Doing the same thing here
-			// for the time being.  --achurch
-			if (core->currentLayerPass != this->renderPass
-			 && core->currentLayerPass != this->overrideRenderPass)
-				return;
-		}
-		else if (top != NULL && top->overrideRenderPass != OVERRIDE_NONE)
-		{
-			if (core->currentLayerPass != top->overrideRenderPass)
-				return;
-		}
-		else
-		{
-			if (!hasRenderPass(core->currentLayerPass))
-				return;
-		}
-	}
-
 	if (MotionBlurData *mb = this->motionBlur)
 	{
 		RenderState rx(rs);
@@ -595,9 +599,9 @@ void RenderObject::renderCall(const RenderState& rs) const
 
 
 	bool doRender = true;
-	int pass = renderPass;
-	if (core->currentLayerPass != RENDER_ALL && renderPass != RENDER_ALL)
+	if (rs.pass != RENDER_ALL)
 	{
+		int pass = renderPass;
 		RenderObject *top = getTopParent();
 		if (top)
 		{
@@ -605,7 +609,7 @@ void RenderObject::renderCall(const RenderState& rs) const
 				pass = top->overrideRenderPass;
 		}
 
-		doRender = (core->currentLayerPass == pass);
+		doRender = (rs.pass == pass);
 	}
 
 	if (doRender)

BBGE/RenderObject.h

@@ -111,6 +111,9 @@ public:
 	bool isDead() const {return _dead;}
 	bool isHidden() const {return _hidden || (parent && parent->isHidden());}
 
+	bool shouldTryToRender() const; // somewhat expensive
+	bool isVisibleInPass(int pass) const;
+
 	// Set whether the object is hidden.  If hidden, no updates (except
 	// lifetime checks) or render operations will be performed, and no
 	// child objects will be updated or rendered.
@@ -221,7 +224,6 @@ public:
 
 	//-------------------------------- Methods above, fields below
 
-	static RenderObjectLayer *rlayer;
 	static bool renderCollisionShape;
 	static bool renderPaths;
 	static size_t lastTextureApplied;
@@ -267,9 +269,16 @@ public:
 	float decayRate;
 	float maxLife;
 
-
+	// When a root RenderObject has overrideRenderPass set,
+	// the override applies the same pass to ALL RenderObjects in the hierarchy.
+	// For non-root objects, it has no effect.
 	int overrideRenderPass;
+
+	// In layers that have multi-pass rendering enabled, the object will only be rendered
+	// in this pass (single-pass layers always render, regardless of this setting).
 	int renderPass;
+
+
 	float overrideCullRadiusSqr;
 
 

BBGE/RenderObjectLayer.cpp

@@ -210,30 +210,55 @@ void RenderObjectLayer::moveToBack(RenderObject *r)
 	}
 }
 
-void RenderObjectLayer::renderPass(const RenderState& rs, int pass)
-{
-	core->currentLayerPass = pass;
-
-	for (RenderObject *robj = getFirst(); robj; robj = getNext())
-	{
-		renderOneObject(rs, robj);
-	}
-}
-
 void RenderObjectLayer::reloadDevice()
 {
 }
 
-inline void RenderObjectLayer::renderOneObject(const RenderState& rs, const RenderObject *robj)
+void RenderObjectLayer::prepareRender()
 {
-	core->totalRenderObjectCount++;
-	if (robj->getParent() || robj->alpha.x == 0)
+	toRender.clear();
+
+	size_t n = 0;
+	for (const RenderObject *robj = getFirst(); robj; robj = getNext())
+	{
+		++n;
+		if(robj->shouldTryToRender())
+			toRender.push_back(robj);
+	}
+	core->renderObjectCount += toRender.size();
+	toRender.push_back(NULL); // terminate
+	core->totalRenderObjectCount += n;
+}
+
+void RenderObjectLayer::render() const
+{
+	if(toRender.size() <= 1)
 		return;
 
-	if (!robj->cull || robj->isOnScreen())
+	size_t proc = 0;
+	CombinedRenderAndGPUState rs;
+
+	if (startPass == endPass)
 	{
-		robj->render(rs);
-		core->renderObjectCount++;
+		rs.pass = RenderObject::RENDER_ALL;
+		const RenderObject * const * rlist = &toRender[0]; // known to have at least one element
+		while(const RenderObject *ro = *rlist++)
+			ro->render(rs);
+		proc += toRender.size() - 1;
 	}
-	core->processedRenderObjectCount++;
+	else
+	{
+		for (int pass = startPass; pass <= endPass; pass++)
+		{
+			rs.pass = pass;
+			const RenderObject * const * rlist = &toRender[0]; // known to have at least one element
+			while(const RenderObject *ro = *rlist++)
+				if(ro->isVisibleInPass(pass))
+				{
+					ro->render(rs);
+					++proc;
+				}
+		}
+	}
+	core->processedRenderObjectCount += proc;
 }

BBGE/RenderState.cpp

@@ -4,7 +4,7 @@
 
 
 RenderState::RenderState(GPUState &gpu)
-    : gpu(gpu), color(1,1,1), alpha(1)
+    : gpu(gpu), color(1,1,1), alpha(1), pass(0)
 {
 }
 

BBGE/RenderState.h

@@ -27,6 +27,7 @@ struct RenderState
 
     Vector color;
     float alpha;
+    int pass;
 
 protected:
     RenderState(GPUState& gpu);