#define Transparent vec4(.0,.0,.0,.0) #define Tolerance 0.004 //If you feel like your…
- User
#define Transparent vec4(.0,.0,.0,.0) #define Tolerance 0.004 //If you feel like your colors should be matching but aren't, increase this number a bit.
varying vec2 v_vTexcoord; varying vec4 v_vColour;
uniform sampler2D u_palTexture; uniform vec4 u_Uvs; uniform float u_paletteId; uniform vec2 u_pixelSize;
vec4 findAltColor(vec4 inCol, vec2 corner) { if(inCol.a == 0.) return Transparent;
float dist; vec2 testPos; vec4 leftCol; for(float i = corner.y; i < u_Uvs.w; i+=u_pixelSize.y ) { testPos = vec2(corner.x,i); leftCol = texture2D( u_palTexture, testPos); dist = distance(leftCol,inCol); if(dist < Tolerance) { testPos = vec2(corner.x + u_pixelSize.x * floor(u_paletteId + 1.0), i); return mix(texture2D(u_palTexture, vec2(testPos.x - u_pixelSize.x, testPos.y)), texture2D(u_palTexture, testPos), fract(u_paletteId)); } } return inCol;}
void main() { vec4 col = texture2D( gm_BaseTexture, v_vTexcoord); DoAlphaTest( col ); col = findAltColor(col, u_Uvs.xy); gl_FragColor = v_vColour * col; }
attribute vec3 in_Position; // (x,y,z) attribute vec4 in_Colour; // (r,g,b,a) attribute vec2 in_TextureCoord; // (u,v)
varying vec2 v_vTexcoord; varying vec4 v_vColour;
void main() { vec4 object_space_pos = vec4( in_Position.x, in_Position.y, in_Position.z, 1.0); gl_Position = gm_Matrices[MATRIX_WORLD_VIEW_PROJECTION] * object_space_pos;
v_vColour = in_Colour; v_vTexcoord = in_TextureCoord;}
#macro rps_type_standard 0 #macro rps_type_html5_sprite 1 #macro rps_type_html5_surface 2
///@func pal_swap_init_system(shader, [html5 sprite shader], [html5 surface shader]) function pal_swap_init_system(_shader = shd_pal_swapper, _html5_sprite_shader = shd_pal_html_sprite, _html5_surface_shader = shd_pal_html_surface) {
//Initiates the palette system. Call once at the beginning of your game somewhere. //IMPORTANT: if you plan on using this for HTML5, you need to specify your two HTML5//shader assets var _swapper = { shader: noone, html5: false, html5_sprite: noone, html5_surface: noone, texel_size: [0], uvs: [0], index: [0], texture: [0], layer_data: {} }
_swapper.html5 = os_browser != browser_not_a_browser; if(!_swapper.html5) { //Init non-html5 mode _swapper.shader = _shader; _swapper.texel_size[rps_type_standard] = shader_get_uniform(_shader, "u_pixelSize"); _swapper.uvs[rps_type_standard] = shader_get_uniform(_shader, "u_Uvs"); _swapper.index[rps_type_standard] = shader_get_uniform(_shader, "u_paletteId"); _swapper.texture[rps_type_standard] = shader_get_sampler_index(_shader, "u_palTexture"); } else { if(_html5_sprite_shader == undefined || _html5_surface_shader == undefined) { show_message("Must provide pal_swap_init_system() with 2 additional arguments for HTML5 Compatible Sprite and Surface Shaders"); game_end(); } _swapper.html5_sprite = _html5_sprite_shader; _swapper.html5_surface = _html5_surface_shader; _swapper.texel_size[rps_type_html5_sprite] = shader_get_uniform(_html5_sprite_shader, "u_pixelSize"); _swapper.uvs[rps_type_html5_sprite] = shader_get_uniform(_html5_sprite_shader, "u_Uvs"); _swapper.index[rps_type_html5_sprite] = shader_get_uniform(_html5_sprite_shader, "u_paletteId"); _swapper.texture[rps_type_html5_sprite] = shader_get_sampler_index(_html5_sprite_shader, "u_palTexture"); _swapper.texel_size[rps_type_html5_surface] = shader_get_uniform(_html5_surface_shader, "u_pixelSize"); _swapper.uvs[rps_type_html5_surface] = shader_get_uniform(_html5_surface_shader, "u_Uvs"); _swapper.index[rps_type_html5_surface] = shader_get_uniform(_html5_surface_shader, "u_paletteId"); _swapper.texture[rps_type_html5_surface] = shader_get_sampler_index(_html5_surface_shader, "u_palTexture"); } global.retro_pal_swapper = _swapper;} pal_swap_init_system();
///@func pal_swap_set(palette_sprite_index, palette_index, palette is surface); function pal_swap_set(_pal_sprite, _pal_index, _is_surface) {
var _swapper = global.retro_pal_swapper; if(_pal_index == 0) exit; var _mode = rps_type_standard; if(!_is_surface) { //Using a sprite based palette if(_swapper.html5) { shader_set(_swapper.html5_sprite); _mode = rps_type_html5_sprite; } else shader_set(_swapper.shader); var _tex = sprite_get_texture(_pal_sprite, 0), _UVs = sprite_get_uvs(_pal_sprite, 0); texture_set_stage(_swapper.texture[_mode], _tex); var _texel_x = texture_get_texel_width(_tex), _texel_y = texture_get_texel_height(_tex), _texel_hx = _texel_x * 0.5, _texel_hy = _texel_y * 0.5; shader_set_uniform_f(_swapper.texel_size[_mode], _texel_x, _texel_y); shader_set_uniform_f(_swapper.uvs[_mode], _UVs[0] + _texel_hx, _UVs[1] + _texel_hy, _UVs[2], _UVs[3]); shader_set_uniform_f(_swapper.index[_mode], _pal_index); } else { //Using a surface based palette if(_swapper.html5) { shader_set(_swapper.html5_surface); _mode = rps_type_html5_surface; } else shader_set(_swapper.shader); var _tex = surface_get_texture(_pal_sprite); texture_set_stage(_swapper.texture[_mode], _tex); var _texel_x = texture_get_texel_width(_tex), _texel_y = texture_get_texel_height(_tex), _texel_hx = _texel_x * 0.5, _texel_hy = _texel_y * 0.5; shader_set_uniform_f(_swapper.texel_size[_mode], _texel_x, _texel_y); shader_set_uniform_f(_swapper.uvs[_mode], _texel_hx, _texel_hy, 1.0 + _texel_hx, 1.0 + _texel_hy); shader_set_uniform_f(_swapper.index[_mode], _pal_index); }}
/// @func pal_swap_reset(); function pal_swap_reset() { //Resets the shader if(shader_current() != -1){ shader_reset(); } }
///@func pal_swap_layer_init();
// Should be called once before setting up any layers to pal swap. // This should be done on room start function pal_swap_layer_init() { global.retro_pal_swapper.layer_data = {}; }// @func pal_swap_set_layer(pal_sprite, pal_index, layer_index, pal_is_surface) //draws the specified layer using the specified palette in the specified event. function pal_swap_set_layer(_pal_sprite, _pal_index, _layer_index, _pal_is_surface) { global.retro_pal_swapper.layer_data[$ _layer_index] = {sprite: _pal_sprite, index: _pal_index, is_surf: _pal_is_surface}; }
///@func pal_swap_enable_layer(layer index) /// This script sets layer start and end scripts. /// This script should not be called in either a step event or a draw event.
/// It should only be called once. function pal_swap_enable_layer(_layer_index) { if(!layer_exists(_layer_index)) return;var _onLayerBegin = method({layer: _layer_index}, function(){ if(event_type == ev_draw) { var _data = global.retro_pal_swapper.layer_data[$ layer]; if(_data == undefined) return;
pal_swap_set(_data.sprite,_data.index,_data.is_surf); }});
layer_script_begin(_layer_index, _onLayerBegin);
layer_script_end(_layer_index,function() { if(event_type == ev_draw) { pal_swap_reset(); } }); }
pal_swap_batch_begin(spr_roof_palette, false); for (var i = array_length(decorations) -1; i >= 0; --i) { with (decorations[i]){ pal_swap_batch_pack_data(palette_index,1); Draw(); } } pal_swap_batch_end();
I want to use image alpha of the objects to pass both palette + alpha (max 32 color palette so adjust bits accorddingly between alpha and color)
So I can batch draw decorations with a single shader set without breaking vertex batchs in gamemaker studio 2.3. Add a new version my shader and add additional functions to achieve htat.
- Assistant
To achieve your goal of encoding both palette index and alpha into the image alpha channel while maintaining batching in GameMaker Studio 2.3, we'll need to modify the shader and add new functions to handle this. The idea is to use the 8-bit alpha channel to store both a palette index (up to 32 colors, requiring 5 bits) and an alpha value (using the remaining 3 bits for 8 levels of transparency). This will allow you to batch draw decorations without breaking the vertex batch, as the palette index and alpha will be extracted in the shader.
Below, I'll provide the modified shader, new functions, and explanations of the changes.
Step 1: Modify the Shader
We'll modify the fragment shader to decode the palette index and alpha from the texture's alpha channel. The palette index will use the lower 5 bits (0-31 for 32 colors), and the alpha will use the upper 3 bits (0-7, scaled to 0-1).
Here’s the updated shader code:
#define Transparent vec4(.0,.0,.0,.0) #define Tolerance 0.004 // If you feel like your colors should be matching but aren't, increase this number a bit. varying vec2 v_vTexcoord; varying vec4 v_vColour; uniform sampler2D u_palTexture; uniform vec4 u_Uvs; uniform float u_paletteId; uniform vec2 u_pixelSize; vec4 findAltColor(vec4 inCol, vec2 corner) { if(inCol.a == 0.) return Transparent; float dist; vec2 testPos; vec4 leftCol; for(float i = corner.y; i < u_Uvs.w; i+=u_pixelSize.y ) { testPos = vec2(corner.x,i); leftCol = texture2D( u_palTexture, testPos); dist = distance(leftCol,inCol); if(dist < Tolerance) { testPos = vec2(corner.x + u_pixelSize.x * floor(u_paletteId + 1.0), i); return mix(texture2D(u_palTexture, vec2(testPos.x - u_pixelSize.x, testPos.y)), texture2D(u_palTexture, testPos), fract(u_paletteId)); } } return inCol; } // New function to decode palette index and alpha from a single alpha value void decodePaletteAndAlpha(float encodedAlpha, out float paletteIndex, out float alpha) { // Extract palette index (lower 5 bits, 0-31) paletteIndex = floor(encodedAlpha * 255.0) - (floor(floor(encodedAlpha * 255.0) / 32.0) * 32.0); // Extract alpha (upper 3 bits, 0-7, scaled to 0-1) alpha = floor(floor(encodedAlpha * 255.0) / 32.0) / 7.0; } void main() { vec4 col = texture2D( gm_BaseTexture, v_vTexcoord); DoAlphaTest( col ); // If alpha is 0, output transparent if (col.a == 0.0) { gl_FragColor = Transparent; return; } // Decode palette index and alpha from the texture's alpha channel float paletteIndex, alpha; decodePaletteAndAlpha(col.a, paletteIndex, alpha); // If palette index is 0, treat as no palette swap (use original color with decoded alpha) if (paletteIndex == 0.0) { gl_FragColor = v_vColour * vec4(col.rgb, alpha); return; } // Otherwise, perform palette swap using the decoded palette index col = findAltColor(col, u_Uvs.xy); gl_FragColor = v_vColour * vec4(col.rgb, alpha); }Explanation of Changes:
-
New
decodePaletteAndAlphaFunction:- This function takes the encoded alpha value (0-1) and decodes it into a palette index (0-31) and an alpha value (0-7, scaled to 0-1).
- The palette index is stored in the lower 5 bits of the 8-bit alpha value, allowing for 32 possible palette indices.
- The alpha is stored in the upper 3 bits, allowing for 8 discrete alpha levels (0, 0.142, 0.285, 0.428, 0.571, 0.714, 0.857, 1.0).
-
Modified
mainFunction:- We decode the palette index and alpha from the texture's alpha channel.
- If the palette index is 0, we skip palette swapping and use the original color with the decoded alpha.
- Otherwise, we perform the palette swap and apply the decoded alpha.
Step 2: Add New Functions to Encode Palette Index and Alpha
We need to add functions to encode the palette index and alpha into a single alpha value. These functions will be used when preparing the sprites or surfaces for drawing.
Here are the new functions to add to your script:
/// @func pal_swap_encode_alpha(palette_index, alpha) /// @desc Encodes a palette index (0-31) and alpha (0-1) into a single alpha value (0-1). /// @param palette_index The palette index (0-31). /// @param alpha The alpha value (0-1, will be quantized to 8 levels). /// @return The encoded alpha value (0-1). function pal_swap_encode_alpha(_palette_index, _alpha) { // Clamp palette index to 0-31 _palette_index = clamp(floor(_palette_index), 0, 31); // Quantize alpha to 8 levels (0-7) and clamp to 0-1 var quantized_alpha = clamp(floor(_alpha * 7 + 0.5), 0, 7); // Encode into a single 8-bit value (palette index in lower 5 bits, alpha in upper 3 bits) var encoded_value = (quantized_alpha * 32) + _palette_index; // Normalize to 0-1 return encoded_value / 255.0; } /// @func pal_swap_batch_begin(pal_sprite, pal_is_surface) /// @desc Begins a batch of palette-swapped draws. /// @param pal_sprite The palette sprite or surface. /// @param pal_is_surface Whether the palette is a surface. function pal_swap_batch_begin(_pal_sprite, _pal_is_surface) { var _swapper = global.retro_pal_swapper; var _mode = rps_type_base; if (!_pal_is_surface) { // Using a sprite-based palette if (_swapper.html5) { shader_set(_swapper.html5_sprite); _mode = rps_type_html5_sprite; } else { shader_set(_swapper.shader); } var _tex = sprite_get_texture(_pal_sprite, 0), _UVs = sprite_get_uvs(_pal_sprite, 0); texture_set_stage(_swapper.texture[_mode], _tex); var _texel_x = texture_get_texel_width(_tex), _texel_y = texture_get_texel_height(_tex), _texel_hx = _texel_x * 0.5, _texel_hy = _texel_y * 0.5; shader_set_uniform_f(_swapper.texel_size[_mode], _texel_x, _texel_y); shader_set_uniform_f(_swapper.uvs[_mode], _UVs[0] + _texel_hx, _UVs[1] + _texel_hy, _UVs[2], _UVs[3]); shader_set_uniform_f(_swapper.index[_mode], 0); // Palette index is encoded in alpha, so set to 0 } else { // Using a surface-based palette if (_swapper.html5) { shader_set(_swapper.html5_surface); _mode = rps_type_html5_surface; } else { shader_set(_swapper.shader); } var _tex = surface_get_texture(_pal_sprite); texture_set_stage(_swapper.texture[_mode], _tex); var _texel_x = texture_get_texel_width(_tex), _texel_y = texture_get_texel_height(_tex), _texel_hx = _texel_x * 0.5, _texel_hy = _texel_y * 0.5; shader_set_uniform_f(_swapper.texel_size[_mode], _texel_x, _texel_y); shader_set_uniform_f(_swapper.uvs[_mode], _texel_hx, _texel_hy, 1.0 + _texel_hx, 1.0 + _texel_hy); shader_set_uniform_f(_swapper.index[_mode], 0); // Palette index is encoded in alpha, so set to 0 } } /// @func pal_swap_batch_pack_data(palette_index, alpha) /// @desc Packs palette index and alpha into the current vertex batch. /// @param palette_index The palette index (0-31). /// @param alpha The alpha value (0-1). function pal_swap_batch_pack_data(_palette_index, _alpha) { // Encode palette index and alpha into a single alpha value var encoded_alpha = pal_swap_encode_alpha(_palette_index, _alpha); // Set the vertex color alpha to the encoded value draw_set_alpha(encoded_alpha); } /// @func pal_swap_batch_end() /// @desc Ends a batch of palette-swapped draws. function pal_swap_batch_end() { pal_swap_reset(); draw_set_alpha(1.0); // Reset alpha to default }Explanation of New Functions:
-
pal_swap_encode_alpha:- This function encodes a palette index (0-31) and an alpha value (0-1) into a single alpha value (0-1).
- The palette index is stored in the lower 5 bits, and the alpha is quantized to 8 levels and stored in the upper 3 bits.
-
pal_swap_batch_begin:- This function is similar to
pal_swap_set, but it sets the palette index uniform to 0 because the palette index is now encoded in the alpha channel. - It prepares the shader for batching.
- This function is similar to
-
pal_swap_batch_pack_data:- This function encodes the palette index and alpha into a single alpha value and sets it as the vertex color alpha.
- This ensures that the encoded alpha is passed to the shader without breaking the vertex batch.
-
pal_swap_batch_end:- This function resets the shader and restores the default alpha value.
Step 3: Update Your Drawing Code
Your existing drawing code is already set up to use the batching system, but we need to ensure that the sprites or surfaces used for drawing have their alpha channels set up correctly. Specifically, the alpha channel of the sprite or surface should be set to the encoded alpha value (palette index + alpha).
Here’s how you can update your drawing code:
pal_swap_batch_begin(spr_roof_palette, false); for (var i = array_length(decorations) - 1; i >= 0; --i) { with (decorations[i]) { // Encode palette index and alpha into the alpha channel pal_swap_batch_pack_data(palette_index, image_alpha); Draw(); } } pal_swap_batch_end();Notes on Sprite/Surface Preparation:
- Sprite Preparation: If your sprites are not already set up with the correct alpha values, you may need to preprocess them. You can do this by creating a surface, drawing the sprite with the encoded alpha, and then using the surface as the texture. Alternatively, you can modify the sprite's alpha channel directly in an image editor or via code.
- Surface Preparation: If you're using surfaces, you can draw to the surface with the encoded alpha values directly.
Here’s an example of how to preprocess a sprite to encode the palette index and alpha:
/// @func pal_swap_prepare_sprite(sprite, subimg, palette_index, alpha) /// @desc Prepares a sprite by encoding palette index and alpha into its alpha channel. /// @param sprite The sprite to prepare. /// @param subimg The subimage index. /// @param palette_index The palette index (0-31). /// @param alpha The alpha value (0-1). /// @return The new sprite index with encoded alpha. function pal_swap_prepare_sprite(_sprite, _subimg, _palette_index, _alpha) { var width = sprite_get_width(_sprite); var height = sprite_get_height(_sprite); // Create a surface var surf = surface_create(width, height); surface_set_target(surf); draw_clear_alpha(c_black, 0); // Draw the sprite with encoded alpha var encoded_alpha = pal_swap_encode_alpha(_palette_index, _alpha); draw_sprite_ext(_sprite, _subimg, 0, 0, 1, 1, 0, c_white, encoded_alpha); // Create a new sprite from the surface var new_sprite = sprite_create_from_surface(surf, 0, 0, width, height, false, false, 0, 0); // Clean up surface_reset_target(); surface_free(surf); return new_sprite; }You can use this function to prepare your sprites before drawing them:
// Example: Prepare a sprite for drawing var prepared_sprite = pal_swap_prepare_sprite(spr_decoration, 0, palette_index, image_alpha); pal_swap_batch_begin(spr_roof_palette, false); draw_sprite(prepared_sprite, 0, x, y); pal_swap_batch_end(); // Clean up the prepared sprite sprite_delete(prepared_sprite);
Step 4: Test and Debug
- Test with Different Palette Indices: Ensure that palette indices 0-31 work correctly and that palette index 0 skips palette swapping.
- Test with Different Alpha Values: Ensure that alpha values are quantized correctly to 8 levels and applied properly.
- Check Batching: Use GameMaker's debug tools to ensure that the vertex batch is not broken during drawing.
Step 5: Considerations and Limitations
-
Alpha Quantization:
- The alpha is quantized to 8 levels, which may cause visible banding in smooth alpha transitions. If this is an issue, you could consider using a different bit allocation (e.g., 4 bits for palette index and 4 bits for alpha, reducing the palette size to 16 colors but increasing alpha precision to 16 levels).
-
Sprite/Surface Size:
- Preprocessing sprites or surfaces can increase memory usage, especially if you have many unique sprites. Consider caching prepared sprites if possible.
-
Performance:
- The shader modifications are lightweight and should not significantly impact performance. However, preprocessing sprites or surfaces can be costly, so do it sparingly or cache results.
Final Notes
This solution allows you to batch draw decorations with palette swapping and alpha without breaking the vertex batch in GameMaker Studio 2.3. The key is encoding the palette index and alpha into the alpha channel, which is then decoded in the shader. If you need further adjustments or have specific requirements, let me know!
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.