Texture modified when passed as uniform sampler2D to fragment shader

[mcgeochd]

Hi all,

I have a low level multistage render to texture pipeline where each step is created as its own buffer with its own texture as:

immutableTextureStore = ConfigVariableBool("gl-immutable-texture-storage", False)
immutableTextureStore.setValue(True)

# Setup the engine stuff
engine = pc.GraphicsEngine.get_global_ptr()
pipe = pc.GraphicsPipeSelection.get_global_ptr().make_module_pipe("pandagl")

texW = 512
texH = 256

# Request 8 RGB bits, 8 alpha bits, and no depth buffer.
fb_prop = pc.FrameBufferProperties()
fb_prop.setRgbColor(True)
fb_prop.setRgbaBits(8, 8, 8, 8)
fb_prop.setDepthBits(0)

# Create a WindowProperties object set to size.
win_prop = pc.WindowProperties(size=(texW, texH))

# Don't open a window - force it to be an offscreen buffer.
flags = pc.GraphicsPipe.BF_refuse_window

# Create a GraphicsBuffer to render to, we'll get the textures out of this
finalBuffer = engine.makeOutput(pipe, "Buffer", 0, fb_prop, win_prop, flags)
ftex = pc.Texture("final texture")
ftex.setup2dTexture(texW, texH, pc.Texture.T_unsigned_byte, pc.Texture.F_rgba8)
ftex.setWrapV(pc.Texture.WM_clamp)
ftex.setWrapU(pc.Texture.WM_repeat)
ftex.setClearColor((0,0,0,0))
finalBuffer.addRenderTexture(ftex, pc.GraphicsOutput.RTM_copy_ram)
def makeBuffer(textureName: str):
    buffer = engine.makeOutput(pipe, "Buffer", 0, fb_prop, win_prop, flags, finalBuffer.getGsg(), finalBuffer)
    btex = pc.Texture(textureName)
    btex.setup2dTexture(texW, texH, pc.Texture.T_unsigned_byte, pc.Texture.F_rgba8)
    btex.setWrapV(pc.Texture.WM_clamp)
    btex.setWrapU(pc.Texture.WM_repeat)
    btex.setClearColor((0,0,0,0))
    buffer.addRenderTexture(btex, pc.GraphicsOutput.RTM_copy_ram)
    return buffer, btex

# Create a scene graph, a camera, and a card to render to
cm = pc.CardMaker("card")
def makeScene(textureName: str):
    buffer, btex = makeBuffer(textureName)
    canvas = pc.NodePath("Scene")
    canvas.setDepthTest(False)
    canvas.setDepthWrite(False)
    card = canvas.attachNewNode(cm.generate())
    card.setZ(-1)
    card.setX(-1)
    card.setScale(2)
    cam2D = pc.Camera("Camera")
    lens = pc.OrthographicLens()
    lens.setFilmSize(2, 2)
    lens.setNearFar(0, 1000)
    cam2D.setLens(lens)
    camera = pc.NodePath(cam2D)
    camera.reparentTo(canvas)
    camera.setPos(0, -1, 0)
    display_region = buffer.makeDisplayRegion()
    display_region.camera = camera
    return card, btex

card1, btex1 = makeScene("voronoi")
card2, btex2 = makeScene("boundaries")

The texture that is getting altered is originally created as so:

# Load the shader steps
voronoiShader = pc.Shader.load(pc.Shader.SL_GLSL, vertex="quad.vert", fragment="jumpflood.frag")
boundaryShader = pc.Shader.load(pc.Shader.SL_GLSL, vertex="quad.vert", fragment="boundaries.frag")

def jumpFlood(seeds, sphereXYZ: pc.Texture):
    # Place the seeds in the texture
    texArr = np.zeros((texH, texW, 4), dtype=np.dtype('B'))
    for seed in range(min(seeds, 255)):
        i = np.random.randint(0, texH)
        j = np.random.randint(0, texW)
        texArr[i,j,0] = (j*255) / float(texW)
        texArr[i,j,1] = (i*255) / float(texH)
        texArr[i,j,2] = 1 + seed
        texArr[i,j,3] = 255
    seedsTex = arrayToTexture("seeds", texArr, 'RGBA', pc.Texture.F_rgba8)
    storeTextureAsImage(seedsTex, "seeds")
    # Compute the maximum steps
    N = max(texW, texH)
    maxSteps = int(np.log2(N))
    # Attach shader, load uniforms, and run
    card1.set_shader(voronoiShader)
    card1.set_shader_input("jumpflood", seedsTex)
    card1.set_shader_input("sphereXYZ", sphereXYZ)
    card1.set_shader_input("maxSteps", float(maxSteps))
    card1.set_shader_input("texSize", (float(texW), float(texH)))
    # Start jumping
    for step in range(maxSteps+2):
        card1.set_shader_input("level", step)
        engine.renderFrame()
        card1.set_shader_input("jumpflood", btex1)
    return btex1

btex1 is passed out into another variable which is then passed into the function which modifies it:

voronoi = jumpFlood(seeds, sphereXYZ)
storeTextureAsImage(voronoi, "jumpflood 0")
boundaries, vbs = plateBoundaries(seeds, 2, voronoi)

The method plateBoundaries() accepts voronoi and passes it into a new shader, using a new card and a new buffer:

def plateBoundaries(seeds, smoothingSteps: int, voronoi: pc.Texture):
    # Generate velocity bearings
    velocityBearings = [0]*256
    for i in range(min(seeds, 256)):
        velocityBearings[i] = np.random.random()*2*np.pi
    # Attach shader and load uniforms
    card2.set_shader(boundaryShader)
    card2.set_shader_input("voronoi", voronoi)
    card2.set_shader_input("texSize", (float(texW), float(texH)))
    card2.set_shader_input("velocityBearings", velocityBearings)
    card2.set_shader_input("boundaryBearings", boundaryBearings)
    # Run
    engine.renderFrame()
    storeTextureAsImage(voronoi, "jumpflood 0.5")
    storeTextureAsImage(btex2, "boundaries")
    return btex2, velocityBearings[:seeds]

The fragment shader attached to boundaryShader is 140 lines long, so I won’t paste it all here, but the only lines that interact with voronoi are:

uniform sampler2D voronoi;
...
void main() {
    if (texture2D(voronoi, texcoord).a > 0.0) {
        p3d_FragColor = vec4(0.0);
        return;
    }
    ...
    float ourplate = texture2D(voronoi, texcoord).b;
    ...
    float theirplate = texture2D(voronoi, texcoord+vec2(0,-up)).b;
    ...
    theirplate = texture2D(voronoi, texcoord+vec2(-right,-up)).b;
    ...
    // 6 more repetitions of the above
}

The code for storeTextureAsImage() is:

def storeTextureAsImage(texture: pc.Texture, filename: str):
    frame = pc.PNMImage()
    if not texture.hasRamImage():
        engine.extractTextureData(texture,finalBuffer.get_gsg())
        print (texture.getName(), "missing RAM image, attempting to extract")
    print (texture.getName(), "has RAM image:", texture.hasRamImage())
    print (texture.getName(), "transferred to image:", texture.store(frame))
    print ("image successfully written to disk:", frame.write(path_p3d+filename+".png"), "\n")

---

The crux of the issue is that jumpflood 0.png and jumpflood 0.5.png are different images:
jumpflood 0.png:

jumpflood 0.5.png:

I can’t think of any way that boundaryShader could modify a texture only passed to it as a uniform. I thought maybe a memory barrier issue, but jumpflood 0.png is written to the disk before the render call that results in jumpflood 0.5.png has even been made, before voronoi has even been passed to the shader that will be used to render jumpflood 0.5.png.

Any help is greatly appreciated! This issue has confounded me for several days now.

[Thaumaturge]

Given the mention of “jumping” in the “jumpFlood” method, does the shader named “voronoiShader” iteratively update the texture that it produces? If so, then might it not simply be that you have a call to “renderFrame” between the writing of “jumpflood 0” and “jumpflood 0.5”, causing “voronoiShader” to be run between those two points, and the texture to thus be updated?

[mcgeochd]

I think I see what you mean, buffer1 remains connected so that later calls to renderFrame also update btex1 and therefore voronoi? Is there a way to disconnect previously connected buffers to prevent later stages also rendering to them?

[Thaumaturge]

Hmm… Could you not just perform a single call to “renderFrame” for the whole process? After all, the later stages draw on the earlier, meaning that their data should, theoretically, be ready by the time that they render.

[mcgeochd]

I’m not sure I follow? do you mean a single call to renderFrame for the whole pipeline? If so I don’t think that would work, the pipeline looks like this currently:

0. sphereXYZ is read from a file

1. sphereXYZ, seedsTex > [voronoiShader (x10)] > voronoi

2. voronoi > [boundaryShader (x1)] > boundaries > [boundarySmoothingShader (x2)] > boundaries

3. voronoi, boundaries > [boundaryDistancesShader (x21)] > boundaryDistances

4. voronoi, boundaries > {some processing on the CPU} > oceanSeedsTex

5. sphereXYZ, oceanSeedsTex > [voronoiShader (x10)] > seaDistsTex

This is all a WIP so please excuse the inconsistent use of “Tex” for texture, when its all actually working I’ll clean up variable and method names. Because I’m using jumpflooding, and because I need to do some of the intermediate processing on the CPU, I don’t believe I can execute the entire pipeline in one call.

The reason I’m calling some of the shaders (x10) or (x21) times is because I’m using jumpflooding to calculate the voronoi diagrams and distance fields from arbitrary shapes (and single points). The reason this is more efficient than naively comparing every texel to every seed in one shader pass is that jumpflooding’s time complexity is dependent only on the size of the texture, not on the number of seeds. This is helpful because the final implementation of this will be working on 4K textures, some of which will have upwards of 2.5 million seeds. Even my current test setup with 512x256 textures handles voronoi diagrams with 32k+ seeds.

An explanation of jumpflooding and some shadertoy implementations of it can be found at:

Fast Voronoi Diagrams and Distance Field Textures on the GPU With the Jump...

[Thaumaturge]

Hmm, okay, I see I believe.

In that case… What about having only one stage active at a time, and completing each before beginning the other? Specifically:

• You would have only one card to render to at a given time, either cleaning up the card used by a previous stage when preparing the next one, or reusing that card.
• You would render each stage the full number of times before even setting up the next stage.

This should result in only one thing being rendered at a time, rather than earlier stages re-rendering as you enact subsequent stages.

[mcgeochd]

I think I see what you mean and I could definitely implement it. My one concern is where the intermediate textures get stored. As you can see, voronoi is created in stage 1 but used in stages 2, 3, and 4. If I used the same card wouldn’t voronoi get overwritten in stage 2?

I may be misunderstanding the interaction of the GraphicsBuffers, cards, and SceneGraphs, but currently each SceneGraph, with its own card and camera, is linked to a single GraphicsBuffer via:

display_region = buffer.makeDisplayRegion()
display_region.camera = camera

How would sharing one card between buffers work? Each render call would be on the same SceneGraph unless one card can be on multiple SceneGraphs, so I would think earlier textures rendered would be overwritten unless they were stored on the disk and read back in later?

[Thaumaturge]

I would suggest just keeping one display region, and thus one buffer. As to the texture itself, I think that you should be able to copy it into a new texture, before clearing it for reuse.

[mcgeochd]

I just tried switching all calls to use card1 and btex1, so the relevant parts of stages 1 and 2 now look like:

# Attach shader, load uniforms, and run
card1.set_shader(voronoiShader)
card1.set_shader_input("jumpflood", seedsTex)
card1.set_shader_input("sphereXYZ", sphereXYZ)
card1.set_shader_input("maxSteps", float(maxSteps))
card1.set_shader_input("texSize", (float(texW), float(texH)))
# Start jumping
for step in range(maxSteps+2):
    card1.set_shader_input("level", step)
    engine.renderFrame()
    card1.set_shader_input("jumpflood", btex1)
return btex1.makeCopy()

and

# Attach shader and load uniforms
card1.set_shader(boundaryShader)
card1.set_shader_input("voronoi", voronoi)
card1.set_shader_input("texSize", (float(texW), float(texH)))
card1.set_shader_input("velocityBearings", velocityBearings)
card1.set_shader_input("boundaryBearings", boundaryBearings)
# Run
engine.renderFrame()
storeTextureAsImage(voronoi, "jumpflood 1")
storeTextureAsImage(btex1, "boundaries")
# Do smoothing
card1.set_shader(boundarySmoothingShader)
card1.set_shader_input("boundaries", btex1)
card1.set_shader_input("texSize", (float(texW), float(texH)))
for i in range(smoothingSteps):
    engine.renderFrame()
return btex1.makeCopy(), velocityBearings[:seeds]

outside theses two methods, the code looks the same:

voronoi = jumpFlood(seeds, sphereXYZ)
storeTextureAsImage(voronoi, "jumpflood 0")
boundaries, vbs = plateBoundaries(seeds, 2, voronoi)

so now voronoi, because it’s the result of calling makeCopy() on btex1, according to the documentation, should be:

… a new copy of the same Texture. This copy, if applied to geometry, will be copied into texture as a separate texture from the original, so it will be duplicated in texture memory (and may be independently modified if desired).

but that’s not what happens:
jumpflood 0.png

jumpflood 1.png

boundaries.png

Its clear here that voronoi is a reference to btex1, even though makeCopy() is called because when we later render to btex1 again, voronoi and btex1 are identical.

What’s confounded me the most though, and I’ve deleted and reuploaded both jumpflood 1.png and boundaries.png three times to make sure, is that they both appear as exactly the same upload url in discourse:

(upload://btczpBTgLaY35qnL45jpFAxmvzl.png)
(upload://btczpBTgLaY35qnL45jpFAxmvzl.png)

I don’t know if that’s because they point to the same location in memory or what, because they appear in File Explorer as different image files.

[Thaumaturge]

@mcgeochd

Its clear here that voronoi is a reference to btex1, even though makeCopy() is called because when we later render to btex1 again, voronoi and btex1 are identical.

But are you clearing the texture before using it in the second stage? After all, being a copy, it will presumably contain the data that the original had.

The other thought is that, if I recall correctly, my original idea was that the texture initially-created would be the one that was used across stages, with the copies being a means of holding on to the results from previous stages.

If I’m understanding correctly, you’ve done the opposite: used the copies across stages, leaving the original behind. Based on what you said earlier about the texture, I’m not sure that that would work–after all, if you’re operating on a copy, you’re no longer operating on the texture that’s associated with the rendering.

@mcgeochd

What’s confounded me the most though, and I’ve deleted and reuploaded both jumpflood 1.png and boundaries.png three times to make sure, is that they both appear as exactly the same upload url in discourse:

It may be that Discourse uses a relatively-simple approach to image-identification, assuming that if two images have the same name, then they are in fact the same image.

It might be simplest to just rename the files for the purposes of the thread.

[mcgeochd]

@Thaumaturge

The other thought is that, if I recall correctly, my original idea was that the texture initially-created would be the one that was used across stages, with the copies being a means of holding on to the results from previous stages.

I’m not sure I understand what you mean? The purpose of the copies is to be used in later stages, “the texture initially-created would be the one that was used across stages” seems to imply that one texture is passed along through the pipeline, but both stages 3 and 4 require the outputs of both stages 1 and 2, the purpose of making a copy of btex1 after the first stage is to use it in stages 3 and 4 without worrying about stage 2 writing over it first.

Not that that matters,

@Thaumaturge

But are you clearing the texture before using it in the second stage? After all, being a copy, it will presumably contain the data that the original had.

This seems to have done it! Making a copy of btex1, clearing btex1 with clearImage(), and then passing the copy out of the method appears to work.

[Thaumaturge]

Ah, I’m glad that you got it working!

Regarding the other point:

@mcgeochd

I’m not sure I understand what you mean? The purpose of the copies is to be used in later stages, “the texture initially-created would be the one that was used across stages” seems to imply that one texture is passed along through the pipeline, but both stages 3 and 4 require the outputs of both stages 1 and 2, the purpose of making a copy of btex1 after the first stage is to use it in stages 3 and 4 without worrying about stage 2 writing over it first.

Ah, I think that I perhaps misunderstood your implementation, above. I think now that we were actually in agreement regarding the implementation! Fair enough!