Refactor DDS loading code

1 files changed, 121 insertions, 161 deletions

diff --git a/modules/dds/texture_loader_dds.cpp b/modules/dds/texture_loader_dds.cpp
index 481bb46c24..7baf8506a6 100644
--- a/modules/dds/texture_loader_dds.cpp
+++ b/modules/dds/texture_loader_dds.cpp
@@ -44,21 +44,32 @@ enum {
 	DDSD_MIPMAPCOUNT = 0x00020000,
 	DDPF_FOURCC = 0x00000004,
 	DDPF_ALPHAPIXELS = 0x00000001,
-	DDPF_INDEXED = 0x00000020,
-	DDPF_RGB = 0x00000040,
+	DDPF_RGB = 0x00000040
 };
 
+enum DDSFourCC {
+	DDFCC_DXT1 = PF_FOURCC("DXT1"),
+	DDFCC_DXT3 = PF_FOURCC("DXT3"),
+	DDFCC_DXT5 = PF_FOURCC("DXT5"),
+	DDFCC_ATI1 = PF_FOURCC("ATI1"),
+	DDFCC_BC4U = PF_FOURCC("BC4U"),
+	DDFCC_ATI2 = PF_FOURCC("ATI2"),
+	DDFCC_BC5U = PF_FOURCC("BC5U"),
+	DDFCC_A2XY = PF_FOURCC("A2XY")
+};
+
+// The legacy bitmasked format names here represent the actual data layout in the files,
+// while their official names are flipped (e.g. RGBA8 layout is officially called ABGR8).
 enum DDSFormat {
 	DDS_DXT1,
 	DDS_DXT3,
 	DDS_DXT5,
 	DDS_ATI1,
 	DDS_ATI2,
-	DDS_A2XY,
 	DDS_BGRA8,
 	DDS_BGR8,
-	DDS_RGBA8, //flipped in dds
-	DDS_RGB8, //flipped in dds
+	DDS_RGBA8,
+	DDS_RGB8,
 	DDS_BGR5A1,
 	DDS_BGR565,
 	DDS_BGR10A2,
@@ -70,28 +81,26 @@ enum DDSFormat {
 struct DDSFormatInfo {
 	const char *name = nullptr;
 	bool compressed = false;
-	bool palette = false;
 	uint32_t divisor = 0;
 	uint32_t block_size = 0;
 	Image::Format format = Image::Format::FORMAT_BPTC_RGBA;
 };
 
 static const DDSFormatInfo dds_format_info[DDS_MAX] = {
-	{ "DXT1/BC1", true, false, 4, 8, Image::FORMAT_DXT1 },
-	{ "DXT3/BC2", true, false, 4, 16, Image::FORMAT_DXT3 },
-	{ "DXT5/BC3", true, false, 4, 16, Image::FORMAT_DXT5 },
-	{ "ATI1/BC4", true, false, 4, 8, Image::FORMAT_RGTC_R },
-	{ "ATI2/3DC/BC5", true, false, 4, 16, Image::FORMAT_RGTC_RG },
-	{ "A2XY/DXN/BC5", true, false, 4, 16, Image::FORMAT_RGTC_RG },
-	{ "BGRA8", false, false, 1, 4, Image::FORMAT_RGBA8 },
-	{ "BGR8", false, false, 1, 3, Image::FORMAT_RGB8 },
-	{ "RGBA8", false, false, 1, 4, Image::FORMAT_RGBA8 },
-	{ "RGB8", false, false, 1, 3, Image::FORMAT_RGB8 },
-	{ "BGR5A1", false, false, 1, 2, Image::FORMAT_RGBA8 },
-	{ "BGR565", false, false, 1, 2, Image::FORMAT_RGB8 },
-	{ "BGR10A2", false, false, 1, 4, Image::FORMAT_RGBA8 },
-	{ "GRAYSCALE", false, false, 1, 1, Image::FORMAT_L8 },
-	{ "GRAYSCALE_ALPHA", false, false, 1, 2, Image::FORMAT_LA8 }
+	{ "DXT1/BC1", true, 4, 8, Image::FORMAT_DXT1 },
+	{ "DXT3/BC2", true, 4, 16, Image::FORMAT_DXT3 },
+	{ "DXT5/BC3", true, 4, 16, Image::FORMAT_DXT5 },
+	{ "ATI1/BC4", true, 4, 8, Image::FORMAT_RGTC_R },
+	{ "ATI2/A2XY/BC5", true, 4, 16, Image::FORMAT_RGTC_RG },
+	{ "BGRA8", false, 1, 4, Image::FORMAT_RGBA8 },
+	{ "BGR8", false, 1, 3, Image::FORMAT_RGB8 },
+	{ "RGBA8", false, 1, 4, Image::FORMAT_RGBA8 },
+	{ "RGB8", false, 1, 3, Image::FORMAT_RGB8 },
+	{ "BGR5A1", false, 1, 2, Image::FORMAT_RGBA8 },
+	{ "BGR565", false, 1, 2, Image::FORMAT_RGB8 },
+	{ "BGR10A2", false, 1, 4, Image::FORMAT_RGBA8 },
+	{ "GRAYSCALE", false, 1, 1, Image::FORMAT_L8 },
+	{ "GRAYSCALE_ALPHA", false, 1, 2, Image::FORMAT_LA8 }
 };
 
 Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress, CacheMode p_cache_mode) {
@@ -121,15 +130,14 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 	/* uint32_t depth = */ f->get_32();
 	uint32_t mipmaps = f->get_32();
 
-	//skip 11
+	// Skip reserved.
 	for (int i = 0; i < 11; i++) {
 		f->get_32();
 	}
 
-	//validate
-
+	// Validate.
 	// We don't check DDSD_CAPS or DDSD_PIXELFORMAT, as they're mandatory when writing,
-	// but non-mandatory when reading (as some writers don't set them)...
+	// but non-mandatory when reading (as some writers don't set them).
 	if (magic != DDS_MAGIC || hsize != 124) {
 		ERR_FAIL_V_MSG(Ref<Resource>(), "Invalid or unsupported DDS texture file '" + p_path + "'.");
 	}
@@ -145,65 +153,81 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 
 	/* uint32_t caps_1 = */ f->get_32();
 	/* uint32_t caps_2 = */ f->get_32();
-	/* uint32_t caps_ddsx = */ f->get_32();
+	/* uint32_t caps_3 = */ f->get_32();
+	/* uint32_t caps_4 = */ f->get_32();
 
-	//reserved skip
-	f->get_32();
+	// Skip reserved.
 	f->get_32();
 
-	/*
-	print_line("DDS width: "+itos(width));
-	print_line("DDS height: "+itos(height));
-	print_line("DDS mipmaps: "+itos(mipmaps));
+	if (f->get_position() < 128) {
+		f->seek(128);
+	}
 
-	printf("fourcc: %x fflags: %x, rgbbits: %x, fsize: %x\n",format_fourcc,format_flags,format_rgb_bits,format_size);
-	printf("rmask: %x gmask: %x, bmask: %x, amask: %x\n",format_red_mask,format_green_mask,format_blue_mask,format_alpha_mask);
-	*/
+	DDSFormat dds_format = DDS_MAX;
 
-	//must avoid this later
-	while (f->get_position() < 128) {
-		f->get_8();
-	}
+	if (format_flags & DDPF_FOURCC) {
+		// FourCC formats.
+		switch (format_fourcc) {
+			case DDFCC_DXT1: {
+				dds_format = DDS_DXT1;
+			} break;
+			case DDFCC_DXT3: {
+				dds_format = DDS_DXT3;
+			} break;
+			case DDFCC_DXT5: {
+				dds_format = DDS_DXT5;
+			} break;
+			case DDFCC_ATI1:
+			case DDFCC_BC4U: {
+				dds_format = DDS_ATI1;
+			} break;
+			case DDFCC_ATI2:
+			case DDFCC_BC5U:
+			case DDFCC_A2XY: {
+				dds_format = DDS_ATI2;
+			} break;
+
+			default: {
+				ERR_FAIL_V_MSG(Ref<Resource>(), "Unrecognized or unsupported FourCC in DDS '" + p_path + "'.");
+			}
+		}
+
+	} else if (format_flags & DDPF_RGB) {
+		// Channel-bitmasked formats.
+		if (format_flags & DDPF_ALPHAPIXELS) {
+			// With alpha.
+			if (format_rgb_bits == 32 && format_red_mask == 0xff0000 && format_green_mask == 0xff00 && format_blue_mask == 0xff && format_alpha_mask == 0xff000000) {
+				dds_format = DDS_BGRA8;
+			} else if (format_rgb_bits == 32 && format_red_mask == 0xff && format_green_mask == 0xff00 && format_blue_mask == 0xff0000 && format_alpha_mask == 0xff000000) {
+				dds_format = DDS_RGBA8;
+			} else if (format_rgb_bits == 16 && format_red_mask == 0x00007c00 && format_green_mask == 0x000003e0 && format_blue_mask == 0x0000001f && format_alpha_mask == 0x00008000) {
+				dds_format = DDS_BGR5A1;
+			} else if (format_rgb_bits == 32 && format_red_mask == 0x3ff00000 && format_green_mask == 0xffc00 && format_blue_mask == 0x3ff && format_alpha_mask == 0xc0000000) {
+				dds_format = DDS_BGR10A2;
+			}
+
+		} else {
+			// Without alpha.
+			if (format_rgb_bits == 24 && format_red_mask == 0xff0000 && format_green_mask == 0xff00 && format_blue_mask == 0xff) {
+				dds_format = DDS_BGR8;
+			} else if (format_rgb_bits == 24 && format_red_mask == 0xff && format_green_mask == 0xff00 && format_blue_mask == 0xff0000) {
+				dds_format = DDS_RGB8;
+			} else if (format_rgb_bits == 16 && format_red_mask == 0x0000f800 && format_green_mask == 0x000007e0 && format_blue_mask == 0x0000001f) {
+				dds_format = DDS_BGR565;
+			}
+		}
 
-	DDSFormat dds_format;
-
-	if (format_flags & DDPF_FOURCC && format_fourcc == PF_FOURCC("DXT1")) {
-		dds_format = DDS_DXT1;
-	} else if (format_flags & DDPF_FOURCC && format_fourcc == PF_FOURCC("DXT3")) {
-		dds_format = DDS_DXT3;
-
-	} else if (format_flags & DDPF_FOURCC && format_fourcc == PF_FOURCC("DXT5")) {
-		dds_format = DDS_DXT5;
-	} else if (format_flags & DDPF_FOURCC && format_fourcc == PF_FOURCC("ATI1")) {
-		dds_format = DDS_ATI1;
-	} else if (format_flags & DDPF_FOURCC && format_fourcc == PF_FOURCC("ATI2")) {
-		dds_format = DDS_ATI2;
-	} else if (format_flags & DDPF_FOURCC && format_fourcc == PF_FOURCC("A2XY")) {
-		dds_format = DDS_A2XY;
-
-	} else if (format_flags & DDPF_RGB && format_flags & DDPF_ALPHAPIXELS && format_rgb_bits == 32 && format_red_mask == 0xff0000 && format_green_mask == 0xff00 && format_blue_mask == 0xff && format_alpha_mask == 0xff000000) {
-		dds_format = DDS_BGRA8;
-	} else if (format_flags & DDPF_RGB && !(format_flags & DDPF_ALPHAPIXELS) && format_rgb_bits == 24 && format_red_mask == 0xff0000 && format_green_mask == 0xff00 && format_blue_mask == 0xff) {
-		dds_format = DDS_BGR8;
-	} else if (format_flags & DDPF_RGB && format_flags & DDPF_ALPHAPIXELS && format_rgb_bits == 32 && format_red_mask == 0xff && format_green_mask == 0xff00 && format_blue_mask == 0xff0000 && format_alpha_mask == 0xff000000) {
-		dds_format = DDS_RGBA8;
-	} else if (format_flags & DDPF_RGB && !(format_flags & DDPF_ALPHAPIXELS) && format_rgb_bits == 24 && format_red_mask == 0xff && format_green_mask == 0xff00 && format_blue_mask == 0xff0000) {
-		dds_format = DDS_RGB8;
-
-	} else if (format_flags & DDPF_RGB && format_flags & DDPF_ALPHAPIXELS && format_rgb_bits == 16 && format_red_mask == 0x00007c00 && format_green_mask == 0x000003e0 && format_blue_mask == 0x0000001f && format_alpha_mask == 0x00008000) {
-		dds_format = DDS_BGR5A1;
-	} else if (format_flags & DDPF_RGB && format_flags & DDPF_ALPHAPIXELS && format_rgb_bits == 32 && format_red_mask == 0x3ff00000 && format_green_mask == 0xffc00 && format_blue_mask == 0x3ff && format_alpha_mask == 0xc0000000) {
-		dds_format = DDS_BGR10A2;
-	} else if (format_flags & DDPF_RGB && !(format_flags & DDPF_ALPHAPIXELS) && format_rgb_bits == 16 && format_red_mask == 0x0000f800 && format_green_mask == 0x000007e0 && format_blue_mask == 0x0000001f) {
-		dds_format = DDS_BGR565;
-	} else if (!(format_flags & DDPF_ALPHAPIXELS) && format_rgb_bits == 8 && format_red_mask == 0xff) {
-		dds_format = DDS_LUMINANCE;
-	} else if ((format_flags & DDPF_ALPHAPIXELS) && format_rgb_bits == 16 && format_red_mask == 0xff && format_alpha_mask == 0xff00) {
-		dds_format = DDS_LUMINANCE_ALPHA;
-	} else if (format_flags & DDPF_INDEXED && format_rgb_bits == 8) {
-		dds_format = DDS_BGR565;
 	} else {
-		//printf("unrecognized fourcc %x format_flags: %x - rgbbits %i - red_mask %x green mask %x blue mask %x alpha mask %x\n", format_fourcc, format_flags, format_rgb_bits, format_red_mask, format_green_mask, format_blue_mask, format_alpha_mask);
+		// Other formats.
+		if (format_flags & DDPF_ALPHAPIXELS && format_rgb_bits == 16 && format_red_mask == 0xff && format_alpha_mask == 0xff00) {
+			dds_format = DDS_LUMINANCE_ALPHA;
+		} else if (!(format_flags & DDPF_ALPHAPIXELS) && format_rgb_bits == 8 && format_red_mask == 0xff) {
+			dds_format = DDS_LUMINANCE;
+		}
+	}
+
+	// No format detected, error.
+	if (dds_format == DDS_MAX) {
 		ERR_FAIL_V_MSG(Ref<Resource>(), "Unrecognized or unsupported color layout in DDS '" + p_path + "'.");
 	}
 
@@ -218,17 +242,17 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 	uint32_t h = height;
 
 	if (info.compressed) {
-		//compressed bc
-
+		// BC compressed.
 		uint32_t size = MAX(info.divisor, w) / info.divisor * MAX(info.divisor, h) / info.divisor * info.block_size;
+
 		ERR_FAIL_COND_V(size != pitch, Ref<Resource>());
 		ERR_FAIL_COND_V(!(flags & DDSD_LINEARSIZE), Ref<Resource>());
 
 		for (uint32_t i = 1; i < mipmaps; i++) {
 			w = MAX(1u, w >> 1);
 			h = MAX(1u, h >> 1);
+
 			uint32_t bsize = MAX(info.divisor, w) / info.divisor * MAX(info.divisor, h) / info.divisor * info.block_size;
-			//printf("%i x %i - block: %i\n",w,h,bsize);
 			size += bsize;
 		}
 
@@ -236,50 +260,8 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 		uint8_t *wb = src_data.ptrw();
 		f->get_buffer(wb, size);
 
-	} else if (info.palette) {
-		//indexed
-		ERR_FAIL_COND_V(!(flags & DDSD_PITCH), Ref<Resource>());
-		ERR_FAIL_COND_V(format_rgb_bits != 8, Ref<Resource>());
-
-		uint32_t size = pitch * height;
-		ERR_FAIL_COND_V(size != width * height * info.block_size, Ref<Resource>());
-
-		uint8_t palette[256 * 4];
-		f->get_buffer(palette, 256 * 4);
-
-		int colsize = 3;
-		for (int i = 0; i < 256; i++) {
-			if (palette[i * 4 + 3] < 255) {
-				colsize = 4;
-			}
-		}
-
-		int w2 = width;
-		int h2 = height;
-
-		for (uint32_t i = 1; i < mipmaps; i++) {
-			w2 = (w2 + 1) >> 1;
-			h2 = (h2 + 1) >> 1;
-			size += w2 * h2 * info.block_size;
-		}
-
-		src_data.resize(size + 256 * colsize);
-		uint8_t *wb = src_data.ptrw();
-		f->get_buffer(wb, size);
-
-		for (int i = 0; i < 256; i++) {
-			int dst_ofs = size + i * colsize;
-			int src_ofs = i * 4;
-			wb[dst_ofs + 0] = palette[src_ofs + 2];
-			wb[dst_ofs + 1] = palette[src_ofs + 1];
-			wb[dst_ofs + 2] = palette[src_ofs + 0];
-			if (colsize == 4) {
-				wb[dst_ofs + 3] = palette[src_ofs + 3];
-			}
-		}
 	} else {
-		//uncompressed generic...
-
+		// Generic uncompressed.
 		uint32_t size = width * height * info.block_size;
 
 		for (uint32_t i = 1; i < mipmaps; i++) {
@@ -288,6 +270,7 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 			size += w * h * info.block_size;
 		}
 
+		// Calculate the space these formats will take up after decoding.
 		if (dds_format == DDS_BGR565) {
 			size = size * 3 / 2;
 		} else if (dds_format == DDS_BGR5A1) {
@@ -298,9 +281,10 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 		uint8_t *wb = src_data.ptrw();
 		f->get_buffer(wb, size);
 
+		// Decode nonstandard formats.
 		switch (dds_format) {
 			case DDS_BGR5A1: {
-				// TO RGBA
+				// To RGBA8.
 				int colcount = size / 4;
 
 				for (int i = colcount - 1; i >= 0; i--) {
@@ -311,13 +295,16 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 					uint8_t b = wb[src_ofs] & 0x1F;
 					uint8_t g = (wb[src_ofs] >> 5) | ((wb[src_ofs + 1] & 0x3) << 3);
 					uint8_t r = (wb[src_ofs + 1] >> 2) & 0x1F;
+
 					wb[dst_ofs + 0] = r << 3;
 					wb[dst_ofs + 1] = g << 3;
 					wb[dst_ofs + 2] = b << 3;
 					wb[dst_ofs + 3] = a ? 255 : 0;
 				}
+
 			} break;
 			case DDS_BGR565: {
+				// To RGB8.
 				int colcount = size / 3;
 
 				for (int i = colcount - 1; i >= 0; i--) {
@@ -327,21 +314,24 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 					uint8_t b = wb[src_ofs] & 0x1F;
 					uint8_t g = (wb[src_ofs] >> 5) | ((wb[src_ofs + 1] & 0x7) << 3);
 					uint8_t r = wb[src_ofs + 1] >> 3;
+
 					wb[dst_ofs + 0] = r << 3;
 					wb[dst_ofs + 1] = g << 2;
-					wb[dst_ofs + 2] = b << 3; //b<<3;
+					wb[dst_ofs + 2] = b << 3;
 				}
 
 			} break;
 			case DDS_BGR10A2: {
-				// TO RGBA
+				// To RGBA8.
 				int colcount = size / 4;
 
-				for (int i = colcount - 1; i >= 0; i--) {
+				for (int i = 0; i < colcount; i++) {
 					int ofs = i * 4;
 
 					uint32_t w32 = uint32_t(wb[ofs + 0]) | (uint32_t(wb[ofs + 1]) << 8) | (uint32_t(wb[ofs + 2]) << 16) | (uint32_t(wb[ofs + 3]) << 24);
 
+					// This method follows the 'standard' way of decoding 10-bit dds files,
+					// which means the ones created with DirectXTex will be loaded incorrectly.
 					uint8_t a = (w32 & 0xc0000000) >> 24;
 					uint8_t r = (w32 & 0x3ff00000) >> 22;
 					uint8_t g = (w32 & 0xffc00) >> 12;
@@ -350,10 +340,12 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 					wb[ofs + 0] = r;
 					wb[ofs + 1] = g;
 					wb[ofs + 2] = b;
-					wb[ofs + 3] = a == 0xc0 ? 255 : a; //0xc0 should be opaque
+					wb[ofs + 3] = a == 0xc0 ? 255 : a; // 0xc0 should be opaque.
 				}
+
 			} break;
 			case DDS_BGRA8: {
+				// To RGBA8.
 				int colcount = size / 4;
 
 				for (int i = 0; i < colcount; i++) {
@@ -362,44 +354,12 @@ Ref<Resource> ResourceFormatDDS::load(const String &p_path, const String &p_orig
 
 			} break;
 			case DDS_BGR8: {
+				// To RGB8.
 				int colcount = size / 3;
 
 				for (int i = 0; i < colcount; i++) {
 					SWAP(wb[i * 3 + 0], wb[i * 3 + 2]);
 				}
-			} break;
-			case DDS_RGBA8: {
-				/* do nothing either
-				int colcount = size/4;
-
-				for(int i=0;i<colcount;i++) {
-					uint8_t r = wb[i*4+1];
-					uint8_t g = wb[i*4+2];
-					uint8_t b = wb[i*4+3];
-					uint8_t a = wb[i*4+0];
-
-					wb[i*4+0]=r;
-					wb[i*4+1]=g;
-					wb[i*4+2]=b;
-					wb[i*4+3]=a;
-				}
-				*/
-			} break;
-			case DDS_RGB8: {
-				// do nothing
-				/*
-				int colcount = size/3;
-
-				for(int i=0;i<colcount;i++) {
-					SWAP( wb[i*3+0],wb[i*3+2] );
-				}*/
-			} break;
-			case DDS_LUMINANCE: {
-				// do nothing i guess?
-
-			} break;
-			case DDS_LUMINANCE_ALPHA: {
-				// do nothing i guess?
 
 			} break;