1412: support fence textures with "{" prefix and MF_HOLEY .mdl flag f…

…or alpha-masked skins (TrenchBroom#2233)

* 1412: support fence textures with "{" prefix and MF_HOLEY .mdl flag
for alpha-masked skins.

Fixes TrenchBroom#1412

* 1412: silence warning

* 1412: fix test compilation

* 1412: silence warning

* 1412: address review comments

* 1412: use GL_GENERATE_MIPMAP to autogenerate mipmaps

* Revert "1412: use GL_GENERATE_MIPMAP to autogenerate mipmaps"

This reverts commit 44a0f9f.

* 1412: only autogenerate mipmaps if there are none, or it's a masked texture

app/resources/shader/EntityModel.fragsh

@@ -27,6 +27,13 @@ uniform bool GrayScale;
 
 void main() {
     vec4 texel = texture2D(Texture, gl_TexCoord[0].st);
+
+    // Assume alpha masked or opaque.
+    // TODO: Make this optional if we gain support for translucent textures
+    if (texel.a < 0.5) {
+        discard;
+    }
+
     gl_FragColor = vec4(vec3(Brightness / 2.0 * texel), texel.a);
     gl_FragColor = clamp(2 * gl_FragColor, 0.0, 1.0);
 

app/resources/shader/Face.fragsh

@@ -45,6 +45,12 @@ void main() {
 	else
 		gl_FragColor = faceColor;
 
+    // Assume alpha masked or opaque.
+    // TODO: Make this optional if we gain support for translucent textures
+    if (gl_FragColor.a < 0.5) {
+        discard;
+    }
+
     gl_FragColor = vec4(vec3(Brightness / 2.0 * gl_FragColor), gl_FragColor.a);
     gl_FragColor = clamp(2.0 * gl_FragColor, 0.0, 1.0);
     gl_FragColor.a = Alpha;

common/src/Assets/Palette.h

@@ -33,6 +33,10 @@ namespace TrenchBroom {
     }
 
     namespace Assets {
+        enum class PaletteTransparency {
+            Opaque, Index255Transparent
+        };
+
         class Palette {
         private:
             class Data {
@@ -44,22 +48,30 @@ namespace TrenchBroom {
                 ~Data();
 
                 template <typename IndexT, typename ColorT>
-                void indexedToRgb(const Buffer<IndexT>& indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbImage, Color& averageColor) const {
-                    indexedToRgb(&indexedImage[0], pixelCount, rgbImage, averageColor);
+                void indexedToRgba(const Buffer<IndexT>& indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbaImage, Color& averageColor, const PaletteTransparency transparency) const {
+                    indexedToRgba(&indexedImage[0], pixelCount, rgbaImage, averageColor, transparency);
                 }
 
                 template <typename IndexT, typename ColorT>
-                void indexedToRgb(const IndexT* indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbImage, Color& averageColor) const {
+                void indexedToRgba(const IndexT* indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbaImage, Color& averageColor, const PaletteTransparency transparency) const {
                     double avg[3];
                     avg[0] = avg[1] = avg[2] = 0.0;
                     for (size_t i = 0; i < pixelCount; ++i) {
                         const size_t index = static_cast<size_t>(static_cast<unsigned char>(indexedImage[i]));
                         assert(index < m_size);
                         for (size_t j = 0; j < 3; ++j) {
                             const unsigned char c = m_data[index * 3 + j];
-                            rgbImage[i * 3 + j] = c;
+                            rgbaImage[i * 4 + j] = c;
                             avg[j] += static_cast<double>(c);
                         }
+                        switch (transparency) {
+                            case PaletteTransparency::Opaque:
+                                rgbaImage[i * 4 + 3] = 0xFF;
+                                break;
+                            case PaletteTransparency::Index255Transparent:
+                                rgbaImage[i * 4 + 3] = (index == 255) ? 0x00 : 0xFF;
+                                break;
+                        }
                     }
 
                     for (size_t i = 0; i < 3; ++i)
@@ -78,13 +90,13 @@ namespace TrenchBroom {
             static Palette loadPcx(IO::MappedFile::Ptr file);
 
             template <typename IndexT, typename ColorT>
-            void indexedToRgb(const Buffer<IndexT>& indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbImage, Color& averageColor) const {
-                m_data->indexedToRgb(indexedImage, pixelCount, rgbImage, averageColor);
+            void indexedToRgba(const Buffer<IndexT>& indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbaImage, Color& averageColor, const PaletteTransparency transparency = PaletteTransparency::Opaque) const {
+                m_data->indexedToRgba(indexedImage, pixelCount, rgbaImage, averageColor, transparency);
             }
 
             template <typename IndexT, typename ColorT>
-            void indexedToRgb(const IndexT* indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbImage, Color& averageColor) const {
-                m_data->indexedToRgb(indexedImage, pixelCount, rgbImage, averageColor);
+            void indexedToRgba(const IndexT* indexedImage, const size_t pixelCount, Buffer<ColorT>& rgbaImage, Color& averageColor, const PaletteTransparency transparency = PaletteTransparency::Opaque) const {
+                m_data->indexedToRgba(indexedImage, pixelCount, rgbaImage, averageColor, transparency);
             }
         };
     }

common/src/Assets/Texture.cpp

@@ -25,16 +25,30 @@
 
 namespace TrenchBroom {
     namespace Assets {
-        void setMipBufferSize(Assets::TextureBuffer::List& buffers, const size_t width, const size_t height) {
+        size_t bytesPerPixelForFormat(const GLenum format) {
+            switch (format) {
+                case GL_RGB:
+                case GL_BGR:
+                    return 3U;
+                case GL_RGBA:
+                    return 4U;
+            }
+            ensure(false, "unknown format");
+            return 0U;
+        }
+
+        void setMipBufferSize(Assets::TextureBuffer::List& buffers, const size_t width, const size_t height, const GLenum format) {
+            const size_t bytesPerPixel = bytesPerPixelForFormat(format);
+
             for (size_t i = 0; i < buffers.size(); ++i) {
                 const size_t div = 1 << i;
-                const size_t size = 3 * (width * height) / (div * div);
+                const size_t size = bytesPerPixel * (width * height) / (div * div);
                 assert(size > 0);
                 buffers[i] = Assets::TextureBuffer(size);
             }
         }
 
-        Texture::Texture(const String& name, const size_t width, const size_t height, const Color& averageColor, const TextureBuffer& buffer, const GLenum format) :
+        Texture::Texture(const String& name, const size_t width, const size_t height, const Color& averageColor, const TextureBuffer& buffer, const GLenum format, const TextureType type) :
         m_collection(nullptr),
         m_name(name),
         m_width(width),
@@ -43,14 +57,15 @@ namespace TrenchBroom {
         m_usageCount(0),
         m_overridden(false),
         m_format(format),
+        m_type(type),
         m_textureId(0) {
             assert(m_width > 0);
             assert(m_height > 0);
-            assert(buffer.size() >= m_width * m_height * 3);
+            assert(buffer.size() >= m_width * m_height * bytesPerPixelForFormat(format));
             m_buffers.push_back(buffer);
         }
 
-        Texture::Texture(const String& name, const size_t width, const size_t height, const Color& averageColor, const TextureBuffer::List& buffers, const GLenum format) :
+        Texture::Texture(const String& name, const size_t width, const size_t height, const Color& averageColor, const TextureBuffer::List& buffers, const GLenum format, const TextureType type) :
         m_collection(nullptr),
         m_name(name),
         m_width(width),
@@ -59,16 +74,17 @@ namespace TrenchBroom {
         m_usageCount(0),
         m_overridden(false),
         m_format(format),
+        m_type(type),
         m_textureId(0),
         m_buffers(buffers) {
             assert(m_width > 0);
             assert(m_height > 0);
             for (size_t i = 0; i < m_buffers.size(); ++i) {
-                assert(m_buffers[i].size() >= (m_width * m_height) / ((1 << i) * (1 << i)) * 3);
+                assert(m_buffers[i].size() >= (m_width * m_height) / ((1 << i) * (1 << i)) * bytesPerPixelForFormat(format));
             }
         }
 
-        Texture::Texture(const String& name, const size_t width, const size_t height, const GLenum format) :
+        Texture::Texture(const String& name, const size_t width, const size_t height, const GLenum format, const TextureType type) :
         m_collection(nullptr),
         m_name(name),
         m_width(width),
@@ -77,6 +93,7 @@ namespace TrenchBroom {
         m_usageCount(0),
         m_overridden(false),
         m_format(format),
+        m_type(type),
         m_textureId(0) {}
 
         Texture::~Texture() {
@@ -142,12 +159,22 @@ namespace TrenchBroom {
             glAssert(glPixelStorei(GL_UNPACK_ALIGNMENT, 1));
 
             glAssert(glBindTexture(GL_TEXTURE_2D, textureId));
-            glAssert(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAX_LEVEL, static_cast<GLint>(m_buffers.size() - 1)));
             glAssert(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, minFilter));
             glAssert(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, magFilter));
             glAssert(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT));
             glAssert(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT));
-
+
+            // Quake fence textures tend to have nonsense mipmaps
+            // Also generate mipmaps if we don't have any
+            const bool generateMipmaps =
+                    (m_type == TextureType::Masked) || (m_buffers.size() == 1);
+
+            if (generateMipmaps) {
+                glAssert(glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP, GL_TRUE));
+            } else {
+                glAssert(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAX_LEVEL, static_cast<GLint>(m_buffers.size() - 1)));
+            }
+
             /* Uncomment this and the assignments below to rescale npot textures to pot images before uploading them.
             const size_t potWidth = Math::nextPOT(m_width);
             const size_t potHeight = Math::nextPOT(m_height);
@@ -158,7 +185,10 @@ namespace TrenchBroom {
 
             size_t mipWidth = m_width; //potWidth;
             size_t mipHeight = m_height; //potHeight;
-            for (size_t j = 0; j < m_buffers.size(); ++j) {
+
+            const auto mipmapsToUpload = generateMipmaps ? 1u : m_buffers.size();
+
+            for (size_t j = 0; j < mipmapsToUpload; ++j) {
                 const GLvoid* data = reinterpret_cast<const GLvoid*>(m_buffers[j].ptr());
                 glAssert(glTexImage2D(GL_TEXTURE_2D, static_cast<GLint>(j), GL_RGBA,
                                       static_cast<GLsizei>(mipWidth),

common/src/Assets/Texture.h

@@ -33,8 +33,14 @@ namespace TrenchBroom {
         class TextureCollection;
 
         typedef Buffer<unsigned char> TextureBuffer;
-        void setMipBufferSize(TextureBuffer::List& buffers, const size_t width, const size_t height);
-
+
+        enum class TextureType {
+            Opaque, Masked
+        };
+
+        size_t bytesPerPixelForFormat(GLenum format);
+        void setMipBufferSize(TextureBuffer::List& buffers, size_t width, size_t height, GLenum format);
+
         class Texture {
         private:
             TextureCollection* m_collection;
@@ -48,13 +54,14 @@ namespace TrenchBroom {
             bool m_overridden;
 
             GLenum m_format;
+            TextureType m_type;
 
             mutable GLuint m_textureId;
             mutable TextureBuffer::List m_buffers;
         public:
-            Texture(const String& name, const size_t width, const size_t height, const Color& averageColor, const TextureBuffer& buffer, GLenum format = GL_RGB);
-            Texture(const String& name, const size_t width, const size_t height, const Color& averageColor, const TextureBuffer::List& buffers, GLenum format = GL_RGB);
-            Texture(const String& name, const size_t width, const size_t height, GLenum format = GL_RGB);
+            Texture(const String& name, size_t width, size_t height, const Color& averageColor, const TextureBuffer& buffer, GLenum format, TextureType type);
+            Texture(const String& name, size_t width, size_t height, const Color& averageColor, const TextureBuffer::List& buffers, GLenum format, TextureType type);
+            Texture(const String& name, size_t width, size_t height, GLenum format = GL_RGB, TextureType type = TextureType::Opaque);
             ~Texture();
 
             const String& name() const;

common/src/IO/FreeImageTextureReader.cpp

@@ -45,8 +45,9 @@ namespace TrenchBroom {
             const size_t                imageHeight     = static_cast<size_t>(FreeImage_GetHeight(image));
             const FREE_IMAGE_COLOR_TYPE imageColourType = FreeImage_GetColorType(image);
 
+            const auto format = GL_BGR;
             Assets::TextureBuffer::List buffers(4);
-            Assets::setMipBufferSize(buffers, imageWidth, imageHeight);
+            Assets::setMipBufferSize(buffers, imageWidth, imageHeight, format);
 
             // TODO: Alpha channel seems to be unsupported by the Texture class
             if (imageColourType != FIC_RGB) {
@@ -67,7 +68,7 @@ namespace TrenchBroom {
             FreeImage_Unload(image);
             FreeImage_CloseMemory(imageMemory);
 
-            return new Assets::Texture(textureName(imageName, path), imageWidth, imageHeight, Color(), buffers, GL_BGR);
+            return new Assets::Texture(textureName(imageName, path), imageWidth, imageHeight, Color(), buffers, format, Assets::TextureType::Opaque);
         }
     }
 

common/src/IO/IdWalTextureReader.cpp

@@ -46,7 +46,7 @@ namespace TrenchBroom {
             const size_t width = reader.readSize<uint32_t>();
             const size_t height = reader.readSize<uint32_t>();
 
-            Assets::setMipBufferSize(buffers, width, height);
+            Assets::setMipBufferSize(buffers, width, height, GL_RGBA);
 
             for (size_t i = 0; i < MipLevels; ++i)
                 offset[i] = reader.readSize<int32_t>();
@@ -56,12 +56,12 @@ namespace TrenchBroom {
                 const size_t size = mipSize(width, height, i);
                 const char* data = begin + offset[i];
 
-                m_palette.indexedToRgb(data, size, buffers[i], tempColor);
+                m_palette.indexedToRgba(data, size, buffers[i], tempColor);
                 if (i == 0)
                     averageColor = tempColor;
             }
 
-            return new Assets::Texture(textureName(name, path), width, height, averageColor, buffers);
+            return new Assets::Texture(textureName(name, path), width, height, averageColor, buffers, GL_RGBA, Assets::TextureType::Opaque);
         }
     }
 }

common/src/IO/Md2Parser.cpp

@@ -331,10 +331,10 @@ namespace TrenchBroom {
             const ImageLoader image(ImageLoader::PCX, file->begin(), file->end());
 
             const Buffer<unsigned char>& indices = image.indices();
-            Buffer<unsigned char> rgbImage(indices.size() * 3);
-            m_palette.indexedToRgb(indices, indices.size(), rgbImage, avgColor);
+            Buffer<unsigned char> rgbaImage(indices.size() * 4);
+            m_palette.indexedToRgba(indices, indices.size(), rgbaImage, avgColor);
 
-            return new Assets::Texture(skin.name, image.width(), image.height(), avgColor, rgbImage);
+            return new Assets::Texture(skin.name, image.width(), image.height(), avgColor, rgbaImage, GL_RGBA, Assets::TextureType::Opaque);
         }
 
         Assets::Md2Model::FrameList Md2Parser::buildFrames(const Md2FrameList& frames, const Md2MeshList& meshes) {

common/src/IO/MdlParser.cpp

@@ -205,6 +205,7 @@ namespace TrenchBroom {
             Vec3f(-0.688191f, -0.587785f, -0.425325f),
         };
 
+        static const int MF_HOLEY = (1 << 14);
 
         MdlParser::MdlParser(const String& name, const char* begin, const char* end, const Assets::Palette& palette) :
         m_name(name),
@@ -229,8 +230,10 @@ namespace TrenchBroom {
             const size_t skinVertexCount = readSize<int32_t>(cursor);
             const size_t skinTriangleCount = readSize<int32_t>(cursor);
             const size_t frameCount = readSize<int32_t>(cursor);
+            /* const size_t syncType = */ readSize<int32_t>(cursor);
+            const int flags = readInt<int32_t>(cursor);
 
-            parseSkins(cursor, *model, skinCount, skinWidth, skinHeight);
+            parseSkins(cursor, *model, skinCount, skinWidth, skinHeight, flags);
             const MdlSkinVertexList skinVertices = parseSkinVertices(cursor, skinVertexCount);
             const MdlSkinTriangleList skinTriangles = parseSkinTriangles(cursor, skinTriangleCount);
             parseFrames(cursor, *model, frameCount, skinTriangles, skinVertices, skinWidth, skinHeight, origin, scale);
@@ -239,22 +242,28 @@ namespace TrenchBroom {
             return model;
         }
 
-        void MdlParser::parseSkins(const char*& cursor, Assets::MdlModel& model, const size_t count, const size_t width, const size_t height) {
+        void MdlParser::parseSkins(const char*& cursor, Assets::MdlModel& model, const size_t count, const size_t width, const size_t height, const int flags) {
             const size_t size = width * height;
+            const auto transparency = (flags & MF_HOLEY)
+                    ? Assets::PaletteTransparency::Index255Transparent
+                    : Assets::PaletteTransparency::Opaque;
+            const auto type = (transparency == Assets::PaletteTransparency::Index255Transparent)
+                              ? Assets::TextureType::Masked
+                              : Assets::TextureType::Opaque;
             Color avgColor;
             StringStream textureName;
 
             cursor = m_begin + MdlLayout::Skins;
             for (size_t i = 0; i < count; ++i) {
                 const size_t skinGroup = readSize<int32_t>(cursor);
                 if (skinGroup == 0) {
-                    Buffer<unsigned char> rgbImage(size * 3);
-                    m_palette.indexedToRgb(cursor, size, rgbImage, avgColor);
+                    Buffer<unsigned char> rgbaImage(size * 4);
+                    m_palette.indexedToRgba(cursor, size, rgbaImage, avgColor, transparency);
                     cursor += size;
 
                     textureName << m_name << "_" << i;
 
-                    Assets::Texture* texture = new Assets::Texture(textureName.str(), width, height, avgColor, rgbImage);
+                    Assets::Texture* texture = new Assets::Texture(textureName.str(), width, height, avgColor, rgbaImage, GL_RGBA, type);
                     model.addSkin(new Assets::MdlSkin(texture));
                 } else {
                     const size_t pictureCount = readSize<int32_t>(cursor);
@@ -267,14 +276,14 @@ namespace TrenchBroom {
                         cursor = base + j * sizeof(float);
                         times[j] = readFloat<float>(cursor);
 
-                        Buffer<unsigned char> rgbImage(size * 3);
+                        Buffer<unsigned char> rgbaImage(size * 4);
                         cursor = base + pictureCount * 4 + j * size;
-                        m_palette.indexedToRgb(cursor, size, rgbImage, avgColor);
+                        m_palette.indexedToRgba(cursor, size, rgbaImage, avgColor, transparency);
                         cursor += size;
 
                         textureName << m_name << "_" << i << "_" << j;
 
-                        textures[j] = new Assets::Texture(textureName.str(), width, height, avgColor, rgbImage);
+                        textures[j] = new Assets::Texture(textureName.str(), width, height, avgColor, rgbaImage, GL_RGBA, type);
                     }
 
                     model.addSkin(new Assets::MdlSkin(textures, times));

common/src/IO/MdlParser.h

@@ -65,7 +65,7 @@ namespace TrenchBroom {
         private:
             Assets::EntityModel* doParseModel() override;
 
-            void parseSkins(const char*& cursor, Assets::MdlModel& model, const size_t count, const size_t width, const size_t height);
+            void parseSkins(const char*& cursor, Assets::MdlModel& model, size_t count, size_t width, size_t height, int flags);
             MdlSkinVertexList parseSkinVertices(const char*& cursor, const size_t count);
             MdlSkinTriangleList parseSkinTriangles(const char*& cursor, const size_t count);
             void parseFrames(const char*& cursor, Assets::MdlModel& model, const size_t count, const MdlSkinTriangleList& skinTriangles, const MdlSkinVertexList& skinVertices, const size_t skinWidth, const size_t skinHeight, const Vec3f& origin, const Vec3f& scale);