Custom Shaders for GSdx

[Monochrome100]

DolphinFX? What did i miss?

Its shader pack for Dolphin's OpenGL backend would be wonderful if Asmodean updated it, you can download it here: https://forums.dolphin-emu.org/Thread-do...gl-backend

[quequotion]

How can shaders be used in linux?

Yes and no, see gregory's reply.
::UPDATE::
Pretty much no until shaders in the correct format (GLSL; shader model ???) come to exist.

I have been attempting to get Asmodean's GSdx FX shader (latest version) to work with GSdxOGL (Hardware) in pcsx2-git (1.3.0-20140826231313, aka 1.3.0.r5578.9102e40) using a GeForce GT 630 with nvidia drivers 343.13.

Nothing seems to enable custom shaders. Toggling the HOME key (as advised in the wiki, to enable/disable custom shaders) does nothing; toggling the PAGEUP key makes a very minimal improvement in the quality of some jagged edges (the built-in FXAA).

There are multiple questions within this question:

-What is the correct location for shader.fx in linux?
The wiki mentions a cmake var and a directory "-DGLSL_SHADER_DIR="/usr/share/games/pcsx2" in the section on AUR pkgbuild installation. This is the only hint I can find.

I have tried putting shader.fx in:
/usr/bin/ with the executable (not an acceptable thing to do on linux)
~/.config/pcsx2/ because that seems logical
/usr/lib32/pcsx2/ with the plugin libraries
/usr/share/games/pcsx2/ which did not exist (most things are installed into /usr/share/pcsx2 for archlinux)
/usr/share/pcsx2/ instead of /usr/share/games/pcsx2

GSdxOGL cannot (currently) access external shaders.

-Is GSdxOGL compatible with shaders in either Software or Hardware mode?
Has anyone else gotten this to work (in linux)?
All I've achieved so far is causing a bug in my nvidia driver or possibly the gpu itself (not worth attempting to describe; it's consistent across many test runs of pcsx2 with and without attempts to use shader.fx in various locations; possibly caused by simultaneous testing of Asmodean's ComplexMultiShader for FF7 in WINE).

GSdxOGL (currently) cannot be used with external shaders

-Is it necessary to specify some compile-time options to allow the use of custom shaders?

-DREBUILD_SHADER=TRUE

-Is it necessary to replace the built-in fxaa with a custom shader by patching the source?

plugins/GSdx/res/fxaa.fx
However, plugins/GSdx/res/old_fxaa.fx is the shader that (currently) gets compiled in.

[gregory]

For plenty of good reason. There is no support of external shader on GSdx ogl. The variable GLSL_SHADER_DIR is deprecated.

In short if you want to test a new shader, you need to overwrite fxaa.glsl and compile it (-DREBUILD_SHADER=TRUE)

Note: fxaa is still the old fxaa version not the version of Asmodean.

[dokman]

for windows to compile it in the solution waht should i have to do?

[gregory]

lol

There is a perl script => linux_various/glsl2h.pl
Don't ask me how to run this script on windows. Neither if the script work on windows.

[KrossX]

Bump mapping shader by guest.r! Cuz he's too lazy to post it himself.

Code:

/*
  Bump mapping shader coded by guest.r
*/

#ifdef SHADER_MODEL // make safe to include in resource file to enforce dependency

Texture2D Texture;
SamplerState TextureSampler;

static const float glow  = 1.25;  // max brightness on borders
static const float shde  = 0.75;  // max darkening
static const float bump  = 1.33;  // effect strenght - lower values bring more effect
static const float range = 1.00;  // effect width


struct PS_INPUT
{
    float4 p : SV_Position;
    float2 t : TEXCOORD0;
};

struct PS_OUTPUT
{
    float4 c : SV_Target0;
};



float3 TextureSample (float2 texcoord){
    float x = 1.0/1024;
    float y = 1.0/1024;
    float4 yx = float4(x,y,-x,-y)*0.5;

    float3 c11 = Texture.Sample(TextureSampler, texcoord).xyz;
    float3 s00 = Texture.Sample(TextureSampler, texcoord + yx.zw).xyz;
    float3 s20 = Texture.Sample(TextureSampler, texcoord + yx.xw).xyz;
    float3 s22 = Texture.Sample(TextureSampler, texcoord + yx.xy).xyz;
    float3 s02 = Texture.Sample(TextureSampler, texcoord + yx.zy).xyz;
    float3 c00 = Texture.Sample(TextureSampler, texcoord + float2( -x, -y)).xyz;
    float3 c22 = Texture.Sample(TextureSampler, texcoord + float2(  x,  y)).xyz;
    float3 c20 = Texture.Sample(TextureSampler, texcoord + float2(  x, -y)).xyz;
    float3 c02 = Texture.Sample(TextureSampler, texcoord + float2( -x,  y)).xyz;
    float3 c10 = Texture.Sample(TextureSampler, texcoord + float2(0.0, -y)).xyz;
    float3 c21 = Texture.Sample(TextureSampler, texcoord + float2(  x,0.0)).xyz;
    float3 c12 = Texture.Sample(TextureSampler, texcoord + float2(0.0,  y)).xyz;
    float3 c01 = Texture.Sample(TextureSampler, texcoord + float2( -x,0.0)).xyz;  
    float3 dt = float3(1.0,1.0,1.0);

    float d1=dot(abs(c00-c22),dt)+0.001;
    float d2=dot(abs(c20-c02),dt)+0.001;
    float hl=dot(abs(c01-c21),dt)+0.001;
    float vl=dot(abs(c10-c12),dt)+0.001;
    float m1=dot(abs(s00-s22),dt)+0.001;
    float m2=dot(abs(s02-s20),dt)+0.001;

    float3 t1=(hl*(c10+c12)+vl*(c01+c21)+(hl+vl)*c11)/(3.0*(hl+vl));
    float3 t2=(d1*(c20+c02)+d2*(c00+c22)+(d1+d2)*c11)/(3.0*(d1+d2));
  
    c11 =.25*(t1+t2+(m2*(s00+s22)+m1*(s02+s20))/(m1+m2));

    return c11;
}


PS_OUTPUT ps_main(PS_INPUT input)
{
    PS_OUTPUT output;
    float2 pos = input.t;  

    float x = range/2048.0;
    float y = range/2048.0;
    float2 dg1 = float2( x, y); float2 dg2 = float2(-x, y);
    float2 ddx = float2(x,0.0); float2 ddy = float2(0.0,y);

    float3 c11 = TextureSample(pos.xy).xyz;
    float3 c00 = TextureSample(pos.xy - dg1).xyz;
    float3 c22 = TextureSample(pos.xy + dg1).xyz;
    float3 c20 = TextureSample(pos.xy - dg2).xyz;
    float3 c02 = TextureSample(pos.xy + dg2).xyz;
    float3 c10 = TextureSample(pos.xy - ddy).xyz;
    float3 c21 = TextureSample(pos.xy + ddx).xyz;
    float3 c12 = TextureSample(pos.xy + ddy).xyz;
    float3 c01 = TextureSample(pos.xy - ddx).xyz;  
    float3 d11 = c11;

    c11 = (-c00+c22-c01+c21-c10+c12+bump*d11)/bump;
    c11 = min(c11,glow*d11);
    c11 = max(c11,shde*d11);

    output.c.a = 1.0;
    output.c.xyz = c11;

    return output;
}

#endif

[hellbringer616]

Fuax bump mapping eh? I wonder how well it holds up to the real deal, Because this could add some serious improvement

EDIT:
Off


On


I think all it seems to do is make everything blurry

[quequotion]

For plenty of good reason. There is no support of external shader on GSdx ogl.

What are those reasons? This is implemented with lots of things in linux; why not GSdxOGL? Does it have something to do with directx<->ogl translation? file location indecisiveness? preventing people from making false bug reports becausd of botched shader configurations?

I'm not asking to question your judgement, but to get a checklist for a roadmap to a better solution than having to patch and recompile to test each and every new shader or even an adjustment to a shader's parameters.

In short if you want to test a new shader, you need to overwrite fxaa.glsl and compile it (-DREBUILD_SHADER=TRUE)

Wow, I really did not expect the answers to my third and fourth questions to be "yes".

At least it does sound easy enough to do, but having to do it repeatedly would be a chore.

Do you mean plugins/GSdx/res/fxaa.fx?

::UPDATE::
It's easier to replace plugins/GSdx/res/old_fxaa.fx to avoid having to patch both plugins/GSdx/GSDeviceOGL.cpp and pcsx2/linux_various/glsl2h.pl

It's not as easy as plugging in the shaders made for GSdx; (CG)/HLSL shaders need to be converted to GLSL to be compiled into GSdxOGL.

[KrossX]

Revised bump mapping with AA by guest.r

Code:

/*
  Bump mapping shader with AA coded by guest.r
*/

#ifdef SHADER_MODEL // make safe to include in resource file to enforce dependency

Texture2D Texture;
SamplerState TextureSampler;

static const float glow  = 1.25;  // max brightness on borders
static const float shde  = 0.75;  // max darkening
static const float bump  = 1.33;  // effect strenght - lower values bring more effect
static const float range = 1.00;  // effect width


struct PS_INPUT
{
    float4 p : SV_Position;
    float2 t : TEXCOORD0;
};

struct PS_OUTPUT
{
    float4 c : SV_Target0;
};



float3 TextureSample (float2 texcoord){
    float x = 1.0/1024;
    float y = 1.0/1024;
    float4 yx = float4(x,y,-x,-y)*0.5;

    float3 c11 = Texture.Sample(TextureSampler, texcoord).xyz;
    float3 s00 = Texture.Sample(TextureSampler, texcoord + yx.zw).xyz;
    float3 s20 = Texture.Sample(TextureSampler, texcoord + yx.xw).xyz;
    float3 s22 = Texture.Sample(TextureSampler, texcoord + yx.xy).xyz;
    float3 s02 = Texture.Sample(TextureSampler, texcoord + yx.zy).xyz;
    float3 c00 = Texture.Sample(TextureSampler, texcoord + float2( -x, -y)).xyz;
    float3 c22 = Texture.Sample(TextureSampler, texcoord + float2(  x,  y)).xyz;
    float3 c20 = Texture.Sample(TextureSampler, texcoord + float2(  x, -y)).xyz;
    float3 c02 = Texture.Sample(TextureSampler, texcoord + float2( -x,  y)).xyz;
    float3 c10 = Texture.Sample(TextureSampler, texcoord + float2(0.0, -y)).xyz;
    float3 c21 = Texture.Sample(TextureSampler, texcoord + float2(  x,0.0)).xyz;
    float3 c12 = Texture.Sample(TextureSampler, texcoord + float2(0.0,  y)).xyz;
    float3 c01 = Texture.Sample(TextureSampler, texcoord + float2( -x,0.0)).xyz;  
    float3 dt = float3(1.0,1.0,1.0);

    float d1=dot(abs(c00-c22),dt)+0.001;
    float d2=dot(abs(c20-c02),dt)+0.001;
    float hl=dot(abs(c01-c21),dt)+0.001;
    float vl=dot(abs(c10-c12),dt)+0.001;
    float m1=dot(abs(s00-s22),dt)+0.001;
    float m2=dot(abs(s02-s20),dt)+0.001;

    float3 t1=(hl*(c10+c12)+vl*(c01+c21)+(hl+vl)*c11)/(3.0*(hl+vl));
    float3 t2=(d1*(c20+c02)+d2*(c00+c22)+(d1+d2)*c11)/(3.0*(d1+d2));
  
    c11 =.25*(t1+t2+(m2*(s00+s22)+m1*(s02+s20))/(m1+m2));

    return c11;
}


PS_OUTPUT ps_main(PS_INPUT input)
{
    PS_OUTPUT output;
    float2 pos = input.t;  

    float x = range/2048.0;
    float y = range/2048.0;
    float2 dg1 = float2( x, y); float2 dg2 = float2(-x, y);
    float2 ddx = float2(x,0.0); float2 ddy = float2(0.0,y);

    float3 c11 = TextureSample(pos.xy).xyz;
    float3 c00 = TextureSample(pos.xy - dg1).xyz;
    float3 c22 = TextureSample(pos.xy + dg1).xyz;
//    float3 c20 = TextureSample(pos.xy - dg2).xyz;
//    float3 c02 = TextureSample(pos.xy + dg2).xyz;
    float3 c10 = TextureSample(pos.xy - ddy).xyz;
    float3 c21 = TextureSample(pos.xy + ddx).xyz;
    float3 c12 = TextureSample(pos.xy + ddy).xyz;
    float3 c01 = TextureSample(pos.xy - ddx).xyz;  
    float3 d11 = c11;

    c11 = (-c00+c22-c01+c21-c10+c12+bump*d11)/bump;
    c11 = min(c11,glow*d11);
    c11 = max(c11,shde*d11);

    output.c.a = 1.0;
    output.c.xyz = c11;

    return output;
}

#endif

Plain bump mapping by guest.r

Code:

/*
  Bump mapping shader coded by guest.r
*/

#ifdef SHADER_MODEL // make safe to include in resource file to enforce dependency

Texture2D Texture;
SamplerState TextureSampler;

static const float glow  = 1.25;  // max brightness on borders
static const float shde  = 0.75;  // max darkening
static const float bump  = 1.33;  // effect strenght - lower values bring more effect
static const float range = 1.00;  // effect width


struct PS_INPUT
{
    float4 p : SV_Position;
    float2 t : TEXCOORD0;
};

struct PS_OUTPUT
{
    float4 c : SV_Target0;
};


PS_OUTPUT ps_main(PS_INPUT input)
{
    PS_OUTPUT output;
    float2 pos = input.t;  

    float x = range/2048.0;
    float y = range/2048.0;
    float2 dg1 = float2( x, y); float2 dg2 = float2(-x, y);
    float2 ddx = float2(x,0.0); float2 ddy = float2(0.0,y);

    float3 c11 = Texture.Sample(TextureSampler,pos.xy).xyz;
    float3 c00 = Texture.Sample(TextureSampler,pos.xy - dg1).xyz;
    float3 c22 = Texture.Sample(TextureSampler,pos.xy + dg1).xyz;
//    float3 c20 = Texture.Sample(TextureSampler,pos.xy - dg2).xyz;
//    float3 c02 = Texture.Sample(TextureSampler,pos.xy + dg2).xyz;
    float3 c10 = Texture.Sample(TextureSampler,pos.xy - ddy).xyz;
    float3 c21 = Texture.Sample(TextureSampler,pos.xy + ddx).xyz;
    float3 c12 = Texture.Sample(TextureSampler,pos.xy + ddy).xyz;
    float3 c01 = Texture.Sample(TextureSampler,pos.xy - ddx).xyz;  
    float3 d11 = c11;

    c11 = (-c00+c22-c01+c21-c10+c12+bump*d11)/bump;
    c11 = min(c11,glow*d11);
    c11 = max(c11,shde*d11);

    output.c.a = 1.0;
    output.c.xyz = c11;

    return output;
}

#endif

---

Fuax bump mapping eh? I wonder how well it holds up to the real deal, Because this could add some serious improvement

EDIT:
I think all it seems to do is make everything blurry

I did a quick test on that shader and it gave a nice effect on 2d hud elements. I haven't checked the new ones.

[Faendra]

Amazing shader ! I love it ! It's a pleasure to play Final fantasy XII with this
Here some screenshot ^^