Looks like the part of the shader that I posted was wrong anyway, the extra textures did not have lighting.
I came up with this:
result.rgba *=tex0*(1-texA.r)*(1-texA.g)*(1-texA.b)*(1-texA.a)
+texCob*texA.r*(1-texA.g)
+texSand*texA.g
+texRock*texA.b*(1-texA.g)
+texStone*texA.a*(1-texA.b)*(1-texA.g);
It works, but just because I know that cobbles ans sand overlap so I’m multiplying the cobbles with the negative from the sand mask and doing the same with the others.
If I make a new map where cobbles are on stone I will get the artifacts again.
I’m starting to think that trying to fix this in the shader is a bad idea, maybe I should sum, divide or multiply the channels of the mask-texture before I even send it to the shader. Can’t wrap my brain around what I should actually do 
Oh, and here’s the shader I’m using, all of it. I have little to no idea of what I’m actually doing, so if something looks wrong it probably is wrong:
//Cg
/* Generated shader for render state 09E41304:
AlphaTestAttrib:greater,0
LightAttrib:on
render/alight
render/dlight
ShaderAttrib
TexMatrixAttrib: shadow(T:(identity)) ts(T:(identity))
TextureAttrib:on default:gras5 ts:gras5_n shadow: ts:fog3
TexGenAttrib: shadow(world_position) ts(world_position)
TransparencyAttrib:alpha
*/
void vshader(
in float4 vtx_texcoord0 : TEXCOORD0,
out float4 l_texcoord0 : TEXCOORD0,
uniform float4x4 trans_model_to_world,
out float4 l_world_position : TEXCOORD4,
uniform float4x4 trans_model_to_view,
out float4 l_eye_position : TEXCOORD5,
uniform float4x4 tpose_view_to_model,
out float4 l_eye_normal : TEXCOORD6,
in float4 vtx_normal : TEXCOORD4,
out float4 l_tangent : TEXCOORD1,
out float4 l_binormal : TEXCOORD2,
float4 vtx_position : POSITION,
out float4 l_position : POSITION,
uniform float4x4 mat_modelproj
) {
float3 vtx_tangent1= float3(vtx_normal.x, vtx_normal.z, -vtx_normal.y);
float3 vtx_binormal1= float3(vtx_normal.z, vtx_normal.y, -vtx_normal.x);
l_position = mul(mat_modelproj, vtx_position);
l_world_position = mul(trans_model_to_world, vtx_position);
l_eye_position = mul(trans_model_to_view, vtx_position);
l_eye_normal.xyz = mul((float3x3)tpose_view_to_model, vtx_normal.xyz);
l_eye_normal.w = 0;
l_texcoord0 = vtx_texcoord0;
l_tangent.xyz = mul((float3x3)tpose_view_to_model, vtx_tangent1.xyz);
l_tangent.w = 0;
l_binormal.xyz = mul((float3x3)tpose_view_to_model, -vtx_binormal1.xyz);
l_binormal.w = 0;
}
void fshader(
in float4 l_world_position : TEXCOORD4,
in float4 l_eye_position : TEXCOORD5,
in float4 l_eye_normal : TEXCOORD6,
uniform sampler2D tex_0,
in float4 l_texcoord0 : TEXCOORD0,
uniform sampler2D tex_1,
uniform sampler2D tex_2,
uniform float4x4 texmat_2,
uniform sampler2D tex_3,
in uniform sampler2D mask_texture,
in uniform sampler2D rock,
in uniform sampler2D rock_n,
in uniform sampler2D cobble,
in uniform sampler2D cobble_n,
in uniform sampler2D sand,
in uniform sampler2D sand_n,
in uniform sampler2D stone,
in uniform sampler2D stone_n,
uniform float4x4 texmat_3,
in float3 l_tangent : TEXCOORD1,
in float3 l_binormal : TEXCOORD2,
uniform float4 alight_alight0,
uniform float4x4 dlight_dlight0_rel_view,
out float4 o_color : COLOR0,
uniform float4 attr_color,
uniform float4 attr_colorscale
) {
float4 result;
float4 l_texcoord2 = l_world_position;
l_texcoord2 = mul(texmat_2, l_texcoord2);
l_texcoord2.xyz /= l_texcoord2.w;
float4 l_texcoord3 = l_world_position;
l_texcoord3 = mul(texmat_3, l_texcoord3);
l_texcoord3.xyz /= l_texcoord3.w;
// Fetch all textures.
float4 tex0 = tex2D(tex_0, l_texcoord0.xy*64);
float4 tex1 = tex2D(tex_1, l_texcoord0.xy*64); //normal map
float4 tex2 = tex2D(tex_2, l_texcoord2.xy);
float4 tex3 = tex2D(tex_3, l_texcoord3.xy);
float4 texA = tex2D(mask_texture, l_texcoord0.xy);
float4 texRock=tex2D(rock, l_texcoord0.xy*64);
float4 texRock_N=tex2D(rock_n, l_texcoord0.xy*64);
float4 texCob=tex2D(cobble, l_texcoord0.xy*64);
float4 texCob_N=tex2D(cobble_n, l_texcoord0.xy*64);
float4 texSand=tex2D(sand, l_texcoord0.xy*64);
float4 texSand_N=tex2D(sand_n, l_texcoord0.xy*64);
float4 texStone=tex2D(stone, l_texcoord0.xy*64);
float4 texStone_N=tex2D(stone_n, l_texcoord0.xy*64);
float4 tex_normal=tex1*(1-texA.r)*(1-texA.g)*(1-texA.b)*(1-texA.a)
+texCob_N*texA.r*(1-texA.g)
+texSand_N*texA.g
+texRock_N*texA.b*(1-texA.g)
+texStone_N*texA.a*(1-texA.b)*(1-texA.g);
// Translate tangent-space normal in map to view-space.
float3 tsnormal = ((float3)tex_normal * 2) - 1;
l_eye_normal.xyz *= tsnormal.z;
l_eye_normal.xyz += l_tangent * tsnormal.x;
l_eye_normal.xyz += l_binormal * tsnormal.y;
// Correct the surface normal for interpolation effects
l_eye_normal.xyz = normalize(l_eye_normal.xyz);
// Begin view-space light calculations
float ldist,lattenv,langle;
float4 lcolor,lspec,lvec,lpoint,latten,ldir,leye,lhalf;
float4 tot_ambient = float4(0,0,0,0);
float4 tot_diffuse = float4(0,0,0,0);
// Ambient Light 0
lcolor = alight_alight0;
tot_ambient += lcolor;
// Directional Light 0
lcolor = dlight_dlight0_rel_view[0];
lspec = dlight_dlight0_rel_view[1];
lvec = dlight_dlight0_rel_view[2];
lcolor *= saturate(dot(l_eye_normal.xyz, lvec.xyz));
tot_diffuse += lcolor;
// Begin view-space light summation
result = float4(0,0,0,0);
result += tot_ambient;
result += tot_diffuse;
result = saturate(result);
// End view-space light calculations
result.a = 1;
result.rgba *=tex0*(1-texA.r)*(1-texA.g)*(1-texA.b)*(1-texA.a)
+texCob*texA.r*(1-texA.g)
+texSand*texA.g
+texRock*texA.b*(1-texA.g)
+texStone*texA.a*(1-texA.b)*(1-texA.g);
result.rgb = lerp(result, tex2 * float4(0, 0, 0, 1), tex2.r).rgb;
result.rgba *= tex3.rgba;
result *= attr_colorscale;
o_color = result * 1.000001;
}
).
