Fix `Projection::invert` on orthogonal projections and others.

Relates to godotengine/godot#68878, specially when using orthographic projection.

Also adds some tests.

Adapted from godotengine/godot#95303

(cherry picked from commit 07730283d334280e2b4e2bede4902a796eabb4c2)

1 files changed, 219 insertions, 91 deletions

diff --git a/core/math/projection.cpp b/core/math/projection.cpp
index 5e11cbae18..119d0a8142 100644
--- a/core/math/projection.cpp
+++ b/core/math/projection.cpp
@@ -598,101 +598,229 @@ Projection Projection::inverse() const {
 }
 
 void Projection::invert() {
-	int i, j, k;
-	int pvt_i[4], pvt_j[4]; /* Locations of pivot matrix */
-	real_t pvt_val; /* Value of current pivot element */
-	real_t hold; /* Temporary storage */
-	real_t determinant = 1.0f;
-	for (k = 0; k < 4; k++) {
-		/** Locate k'th pivot element **/
-		pvt_val = columns[k][k]; /** Initialize for search **/
-		pvt_i[k] = k;
-		pvt_j[k] = k;
-		for (i = k; i < 4; i++) {
-			for (j = k; j < 4; j++) {
-				if (Math::abs(columns[i][j]) > Math::abs(pvt_val)) {
-					pvt_i[k] = i;
-					pvt_j[k] = j;
-					pvt_val = columns[i][j];
-				}
-			}
-		}
-
-		/** Product of pivots, gives determinant when finished **/
-		determinant *= pvt_val;
-		if (Math::is_zero_approx(determinant)) {
-			return; /** Matrix is singular (zero determinant). **/
-		}
-
-		/** "Interchange" rows (with sign change stuff) **/
-		i = pvt_i[k];
-		if (i != k) { /** If rows are different **/
-			for (j = 0; j < 4; j++) {
-				hold = -columns[k][j];
-				columns[k][j] = columns[i][j];
-				columns[i][j] = hold;
-			}
-		}
-
-		/** "Interchange" columns **/
-		j = pvt_j[k];
-		if (j != k) { /** If columns are different **/
-			for (i = 0; i < 4; i++) {
-				hold = -columns[i][k];
-				columns[i][k] = columns[i][j];
-				columns[i][j] = hold;
-			}
-		}
-
-		/** Divide column by minus pivot value **/
-		for (i = 0; i < 4; i++) {
-			if (i != k) {
-				columns[i][k] /= (-pvt_val);
-			}
-		}
-
-		/** Reduce the matrix **/
-		for (i = 0; i < 4; i++) {
-			hold = columns[i][k];
-			for (j = 0; j < 4; j++) {
-				if (i != k && j != k) {
-					columns[i][j] += hold * columns[k][j];
-				}
-			}
-		}
-
-		/** Divide row by pivot **/
-		for (j = 0; j < 4; j++) {
-			if (j != k) {
-				columns[k][j] /= pvt_val;
-			}
-		}
-
-		/** Replace pivot by reciprocal (at last we can touch it). **/
-		columns[k][k] = 1.0 / pvt_val;
+	// Adapted from Mesa's `src/util/u_math.c` `util_invert_mat4x4`.
+	// MIT licensed. Copyright 2008 VMware, Inc. Authored by Jacques Leroy.
+	Projection temp;
+	real_t *out = (real_t *)temp.columns;
+	real_t *m = (real_t *)columns;
+
+	real_t wtmp[4][8];
+	real_t m0, m1, m2, m3, s;
+	real_t *r0, *r1, *r2, *r3;
+
+#define MAT(m, r, c) (m)[(c) * 4 + (r)]
+
+	r0 = wtmp[0];
+	r1 = wtmp[1];
+	r2 = wtmp[2];
+	r3 = wtmp[3];
+
+	r0[0] = MAT(m, 0, 0);
+	r0[1] = MAT(m, 0, 1);
+	r0[2] = MAT(m, 0, 2);
+	r0[3] = MAT(m, 0, 3);
+	r0[4] = 1.0;
+	r0[5] = 0.0;
+	r0[6] = 0.0;
+	r0[7] = 0.0;
+
+	r1[0] = MAT(m, 1, 0);
+	r1[1] = MAT(m, 1, 1);
+	r1[2] = MAT(m, 1, 2);
+	r1[3] = MAT(m, 1, 3);
+	r1[5] = 1.0;
+	r1[4] = 0.0;
+	r1[6] = 0.0;
+	r1[7] = 0.0;
+
+	r2[0] = MAT(m, 2, 0);
+	r2[1] = MAT(m, 2, 1);
+	r2[2] = MAT(m, 2, 2);
+	r2[3] = MAT(m, 2, 3);
+	r2[6] = 1.0;
+	r2[4] = 0.0;
+	r2[5] = 0.0;
+	r2[7] = 0.0;
+
+	r3[0] = MAT(m, 3, 0);
+	r3[1] = MAT(m, 3, 1);
+	r3[2] = MAT(m, 3, 2);
+	r3[3] = MAT(m, 3, 3);
+
+	r3[7] = 1.0;
+	r3[4] = 0.0;
+	r3[5] = 0.0;
+	r3[6] = 0.0;
+
+	/* choose pivot - or die */
+	if (Math::abs(r3[0]) > Math::abs(r2[0])) {
+		SWAP(r3, r2);
+	}
+	if (Math::abs(r2[0]) > Math::abs(r1[0])) {
+		SWAP(r2, r1);
+	}
+	if (Math::abs(r1[0]) > Math::abs(r0[0])) {
+		SWAP(r1, r0);
+	}
+	ERR_FAIL_COND(0.0 == r0[0]);
+
+	/* eliminate first variable     */
+	m1 = r1[0] / r0[0];
+	m2 = r2[0] / r0[0];
+	m3 = r3[0] / r0[0];
+	s = r0[1];
+	r1[1] -= m1 * s;
+	r2[1] -= m2 * s;
+	r3[1] -= m3 * s;
+	s = r0[2];
+	r1[2] -= m1 * s;
+	r2[2] -= m2 * s;
+	r3[2] -= m3 * s;
+	s = r0[3];
+	r1[3] -= m1 * s;
+	r2[3] -= m2 * s;
+	r3[3] -= m3 * s;
+	s = r0[4];
+	if (s != 0.0) {
+		r1[4] -= m1 * s;
+		r2[4] -= m2 * s;
+		r3[4] -= m3 * s;
+	}
+	s = r0[5];
+	if (s != 0.0) {
+		r1[5] -= m1 * s;
+		r2[5] -= m2 * s;
+		r3[5] -= m3 * s;
+	}
+	s = r0[6];
+	if (s != 0.0) {
+		r1[6] -= m1 * s;
+		r2[6] -= m2 * s;
+		r3[6] -= m3 * s;
+	}
+	s = r0[7];
+	if (s != 0.0) {
+		r1[7] -= m1 * s;
+		r2[7] -= m2 * s;
+		r3[7] -= m3 * s;
 	}
 
-	/* That was most of the work, one final pass of row/column interchange */
-	/* to finish */
-	for (k = 4 - 2; k >= 0; k--) { /* Don't need to work with 1 by 1 corner*/
-		i = pvt_j[k]; /* Rows to swap correspond to pivot COLUMN */
-		if (i != k) { /* If rows are different */
-			for (j = 0; j < 4; j++) {
-				hold = columns[k][j];
-				columns[k][j] = -columns[i][j];
-				columns[i][j] = hold;
-			}
-		}
+	/* choose pivot - or die */
+	if (Math::abs(r3[1]) > Math::abs(r2[1])) {
+		SWAP(r3, r2);
+	}
+	if (Math::abs(r2[1]) > Math::abs(r1[1])) {
+		SWAP(r2, r1);
+	}
+	ERR_FAIL_COND(0.0 == r1[1]);
+
+	/* eliminate second variable */
+	m2 = r2[1] / r1[1];
+	m3 = r3[1] / r1[1];
+	r2[2] -= m2 * r1[2];
+	r3[2] -= m3 * r1[2];
+	r2[3] -= m2 * r1[3];
+	r3[3] -= m3 * r1[3];
+	s = r1[4];
+	if (0.0 != s) {
+		r2[4] -= m2 * s;
+		r3[4] -= m3 * s;
+	}
+	s = r1[5];
+	if (0.0 != s) {
+		r2[5] -= m2 * s;
+		r3[5] -= m3 * s;
+	}
+	s = r1[6];
+	if (0.0 != s) {
+		r2[6] -= m2 * s;
+		r3[6] -= m3 * s;
+	}
+	s = r1[7];
+	if (0.0 != s) {
+		r2[7] -= m2 * s;
+		r3[7] -= m3 * s;
+	}
 
-		j = pvt_i[k]; /* Columns to swap correspond to pivot ROW */
-		if (j != k) { /* If columns are different */
-			for (i = 0; i < 4; i++) {
-				hold = columns[i][k];
-				columns[i][k] = -columns[i][j];
-				columns[i][j] = hold;
-			}
-		}
+	/* choose pivot - or die */
+	if (Math::abs(r3[2]) > Math::abs(r2[2])) {
+		SWAP(r3, r2);
 	}
+	ERR_FAIL_COND(0.0 == r2[2]);
+
+	/* eliminate third variable */
+	m3 = r3[2] / r2[2];
+	r3[3] -= m3 * r2[3];
+	r3[4] -= m3 * r2[4];
+	r3[5] -= m3 * r2[5];
+	r3[6] -= m3 * r2[6];
+	r3[7] -= m3 * r2[7];
+
+	/* last check */
+	ERR_FAIL_COND(0.0 == r3[3]);
+
+	s = 1.0 / r3[3]; /* now back substitute row 3 */
+	r3[4] *= s;
+	r3[5] *= s;
+	r3[6] *= s;
+	r3[7] *= s;
+
+	m2 = r2[3]; /* now back substitute row 2 */
+	s = 1.0 / r2[2];
+	r2[4] = s * (r2[4] - r3[4] * m2);
+	r2[5] = s * (r2[5] - r3[5] * m2);
+	r2[6] = s * (r2[6] - r3[6] * m2);
+	r2[7] = s * (r2[7] - r3[7] * m2);
+	m1 = r1[3];
+	r1[4] -= r3[4] * m1;
+	r1[5] -= r3[5] * m1;
+	r1[6] -= r3[6] * m1;
+	r1[7] -= r3[7] * m1;
+	m0 = r0[3];
+	r0[4] -= r3[4] * m0;
+	r0[5] -= r3[5] * m0;
+	r0[6] -= r3[6] * m0;
+	r0[7] -= r3[7] * m0;
+
+	m1 = r1[2]; /* now back substitute row 1 */
+	s = 1.0 / r1[1];
+	r1[4] = s * (r1[4] - r2[4] * m1);
+	r1[5] = s * (r1[5] - r2[5] * m1),
+	r1[6] = s * (r1[6] - r2[6] * m1);
+	r1[7] = s * (r1[7] - r2[7] * m1);
+	m0 = r0[2];
+	r0[4] -= r2[4] * m0;
+	r0[5] -= r2[5] * m0;
+	r0[6] -= r2[6] * m0;
+	r0[7] -= r2[7] * m0;
+
+	m0 = r0[1]; /* now back substitute row 0 */
+	s = 1.0 / r0[0];
+	r0[4] = s * (r0[4] - r1[4] * m0);
+	r0[5] = s * (r0[5] - r1[5] * m0),
+	r0[6] = s * (r0[6] - r1[6] * m0);
+	r0[7] = s * (r0[7] - r1[7] * m0);
+
+	MAT(out, 0, 0) = r0[4];
+	MAT(out, 0, 1) = r0[5];
+	MAT(out, 0, 2) = r0[6];
+	MAT(out, 0, 3) = r0[7];
+	MAT(out, 1, 0) = r1[4];
+	MAT(out, 1, 1) = r1[5];
+	MAT(out, 1, 2) = r1[6];
+	MAT(out, 1, 3) = r1[7];
+	MAT(out, 2, 0) = r2[4];
+	MAT(out, 2, 1) = r2[5];
+	MAT(out, 2, 2) = r2[6];
+	MAT(out, 2, 3) = r2[7];
+	MAT(out, 3, 0) = r3[4];
+	MAT(out, 3, 1) = r3[5];
+	MAT(out, 3, 2) = r3[6];
+	MAT(out, 3, 3) = r3[7];
+
+#undef MAT
+
+	*this = temp;
 }
 
 void Projection::flip_y() {