How to write a directx shader for 3dsmax

Since programmable GPU was appeared, many and many game engines have to support this feature to create more and more visual effects. And also, 3D modeling softwares need to adapt this new feature.

3dsmax provides a material called "DirectX shader". You can write your directX shader (a.k.a HLSL) and use it as a material in 3dsmax. but there are some rules and restrictions(see DirectX 9 Shader Material-Semantics and Annotations). There are some sample FX files in map\fx\ directory. Let us open the default.fx file.

// This is used by 3dsmax to load the correct parser
string ParamID = "0x0";

//DxMaterial specific
// light direction (view space)
float3 lightDir : Direction < uiname = "Light Direction" object = "TargetLight"> = {-0.577, -0.577, 0.577};

// light intensity
float4 I_a = { 0.1f, 0.1f, 0.1f, 1.0f }; // ambient
float4 I_d = { 1.0f, 1.0f, 1.0f, 1.0f }; // diffuse
float4 I_s = { 1.0f, 1.0f, 1.0f, 1.0f }; // specular

// material reflectivity
float4 k_a < uiname = "Ambient"> = float4( 0.47f, 0.47f, 0.47f, 1.0f ); // ambient
float4 k_d < uiname = "Diffuse"> = float4( 0.47f, 0.47f, 0.47f, 1.0f ); // diffuse
float4 k_s < uiname = "Specular"> = float4( 1.0f, 1.0f, 1.0f, 1.0f ); // diffuse // specular
int n< uiname = "Specular Power" uitype = "IntSpinner" uimin =" 0.0f;" uimax =" 50.0f;"> = 15;

// transformations
float4x4 World : WORLD;
float4x4 View : VIEW;
float4x4 Projection : PROJECTION;
float4x4 WorldViewProj : WORLDVIEWPROJ;
float4x4 WorldView : WORLDVIEW;

struct VS_OUTPUT
{
float4 Pos : POSITION;
float4 col : COLOR0;
};

VS_OUTPUT VS(
float3 Pos : POSITION,
float3 col : COLOR,
float3 Norm : NORMAL,
float2 Tex : TEXCOORD0)
{
VS_OUTPUT Out = (VS_OUTPUT)0;
float3 L = lightDir;
float3 P = mul(float4(Pos, 1),(float4x4)World); // position (view space)
float3 N = normalize(mul(Norm,(float3x3)World)); // normal (view space)
float3 R = normalize(2 * dot(N, L) * N - L); // reflection vector (view space)
float3 V = normalize(P); // view direction (view space)
Out.Pos = mul(float4(Pos,1),WorldViewProj); // position (projected)
float4 Diff = I_a * k_a + I_d * k_d * max(0, dot(N, L)); // diffuse + ambient
float4 Spec = I_s * k_s * pow(max(0, dot(R, V)), n/4); // specular
Out.col = Diff + Spec;
return Out;
}

float4 PS(
float4 Diff : COLOR0,
float4 Spec : COLOR1,
float2 Tex : TEXCOORD0,
float2 Tex1 : TEXCOORD1 ) : COLOR
{
float4 color = Diff + Spec;
return color ;
}

technique DefaultTechnique
{
pass P0
{
// shaders
CullMode = None;
VertexShader = compile vs_1_1 VS();
PixelShader = compile ps_1_1 PS();
}
}

DirectX shader material parses the shader file. These variables with 3dsmax specified annotation will create UI components as you can see in "Default.dx Parameters" rollout.
And 3dsmax will pass values to these variables with specified semantic and annotation.
3dsmax use "Direction" as semantic to specify lighting direction. when there is no light in the scene, this parameter use default light's value i.e. current view direction.

The most trick part is we have to assign all mapping channels to vertex shader inputs if we want to use other coordinates such as vertex color, vertex alpha, second texture coordinate...etc.
Else these input value will be disordered and can not get the correct result.