# feature selection with nonlinear function • ### Question

• Hi everyone

I wrote two methods for feature selection. I have an input vector, a weight vector (which my goal is to learn it) and an output vector. Each element in output vector is a function of some features from input vector.

If this function is linear, for example a weighted sum, it works fine. But when the output is for example a function of the weighted sum it won't work anymore. The weight vector is defined as a mixture of two distributions (one will be chosen for related features and one for non-related features).

Can anyone help me with this?

How can I run my code in case of the non-linear function? What should I change to make this work.

```using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using MicrosoftResearch.Infer;
using MicrosoftResearch.Infer.Distributions;
using MicrosoftResearch.Infer.Maths;
using MicrosoftResearch.Infer.Models;
using MicrosoftResearch.Infer.Factors;

namespace inferNetMassAct
{
class Test
{

public static double testFunction(double x)
{
return (0.25 * x + 0.5 * x * x * x);
}

public static double weightedSum(double[] weightVect, double[] values) {
if(weightVect.Length!= values.Length) return 0;
double weightedSum = 0;
for (int i = 0; i < weightVect.Length; i++ ) {
weightedSum += weightVect[i] * values[i];
}
return weightedSum;
}

public static Variable<double> testFunction(Variable<double> x)
{
return (0.25 * x + 0.5 * x * x * x);
}

static void Main(string[] args)
{
linearFeatureSelection();
//nonlinearFeatureSelection();
}

static void linearFeatureSelection()
{
Console.WriteLine("running linearFeatureSelection() method...\n");
Gaussian dataDist = Gaussian.FromMeanAndPrecision(0, 1);

// training data
int numData = 10;
int numFeatures = 10;
//var rand = new Random();
double[][] data = new double[numData][];
for (int i = 0; i < numData; i++)
{
data[i] = new double[numFeatures];
for (int j = 0; j < data[i].Length; j++)
{
data[i][j] = dataDist.Sample(); //rand.Next(10);
}
}
double[] outputData = new double[numData];
double[] weight = new double[numFeatures];
for (int i = 0; i < weight.Length; i++)
{
if (i < 3)
{
weight[i] = 0.3 * i;
}
else weight[i] = 0;
}

for (int i = 0; i < numData; i++)
{
double weightedSumValue = weightedSum(data[i], weight);
outputData[i] = weightedSumValue + Gaussian.FromMeanAndVariance(0, 0.0025).Sample();// noise is also added
}

// model
Range dataRange = new Range(numData);
Range featureRange = new Range(numFeatures);
VariableArray<VariableArray<double>, double[][]> inputArr =
Variable.Array(Variable.Array<double>(featureRange), dataRange);
VariableArray<double> outputArray = Variable.Array<double>(dataRange).Named("outputArray");
VariableArray<double> weightVar = Variable.Array<double>(featureRange).Named("weights");
VariableArray<bool> selector = Variable.Array<bool>(featureRange).Named("selector");
using (Variable.ForEach(featureRange)) selector[featureRange] = Variable.Bernoulli(0.1);

using (Variable.ForEach(featureRange))
{
using (Variable.If(selector[featureRange]))
{
weightVar[featureRange].SetTo(
Variable.GaussianFromMeanAndPrecision(Variable.Random(Gaussian.FromMeanAndPrecision(1, 0.1)),
Variable.Random(Gamma.FromShapeAndScale(1, 99999)))
);
}
using (Variable.IfNot(selector[featureRange]))
{
weightVar[featureRange].SetTo(Variable.GaussianFromMeanAndVariance(0, 0.0000001).Named("spike"));
}
}
Variable<double> precision = Variable<double>.GammaFromShapeAndScale(1, 99999);
using (Variable.ForEach(dataRange))
{
VariableArray<double> weightSumArray = Variable.Array<double>(featureRange).Named("weightSum");
weightSumArray[featureRange] = (weightVar[featureRange] * inputArr[dataRange][featureRange]).Named("weightSumArray");
// the effect from other nodes - in case of no perturbation
Variable<double> linearEffect =
Variable.Sum(weightSumArray).Named("sum_weightSumArray");

outputArray[dataRange] = Variable.GaussianFromMeanAndPrecision(linearEffect, precision);
}
inputArr.ObservedValue = data;
outputArray.ObservedValue = outputData;
//InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing());
InferenceEngine ie = new InferenceEngine(new ExpectationPropagation());
ie.NumberOfIterations = 50;
//ie.Compiler.GivePriorityTo(typeof(GaussianProductOp_SHG09));
Gaussian[] weightPosterior = ie.Infer<Gaussian[]>(weightVar);
Console.WriteLine("\n*** printing weights posterior...\n");
for (int i = 0; i < weightPosterior.Length; i++) {
Console.WriteLine("weightPosterior["+i+"] = "+weightPosterior[i]);
}

Console.WriteLine("\n\n*** printing selector posterior (Bernouli posterior for feature selection)...\n");
Bernoulli[] selectorPosterior = ie.Infer<Bernoulli[]>(selector);
for (int i = 0; i < selectorPosterior.Length; i++)
{
Console.WriteLine("selectorPosterior[" + i + "] = " + selectorPosterior[i]);
}
}

static void nonlinearFeatureSelection()
{
Console.WriteLine("running nonlinearFeatureSelection() method...\n");
Gaussian dataDist = Gaussian.FromMeanAndPrecision(0, 1);

// training data
int numData = 10;
int numFeatures = 10;
//var rand = new Random();
double[][] data = new double[numData][];
for (int i = 0; i < numData; i++)
{
data[i] = new double[numFeatures];
for (int j = 0; j < data[i].Length; j++)
{
}
}
double[] outputData = new double[numData];
double[] weight = new double[numFeatures];
for (int i = 0; i < weight.Length; i++)
{
if (i < 3)
{
weight[i] = 0.3 * i;
}
else weight[i] = 0;
}

for (int i = 0; i < numData; i++)
{
double weightedSumValue = weightedSum(data[i], weight);
outputData[i] = testFunction(weightedSumValue)+Gaussian.FromMeanAndVariance(0, 0.0025).Sample();// noise is also added
}

// model
Range dataRange = new Range(numData);
Range featureRange = new Range(numFeatures);
VariableArray<VariableArray<double>, double[][]> inputArr =
Variable.Array(Variable.Array<double>(featureRange), dataRange);
VariableArray<double> outputArray = Variable.Array<double>(dataRange).Named("outputArray");
VariableArray<double> weightVar = Variable.Array<double>(featureRange).Named("weights");
VariableArray<bool> selector = Variable.Array<bool>(featureRange).Named("selector");
using (Variable.ForEach(featureRange)) selector[featureRange] = Variable.Bernoulli(0.1);

using (Variable.ForEach(featureRange))
{
using (Variable.If(selector[featureRange]))
{
weightVar[featureRange].SetTo(
Variable.GaussianFromMeanAndPrecision(Variable.Random(Gaussian.FromMeanAndPrecision(1, 0.1)),
Variable.Random(Gamma.FromShapeAndScale(1, 99999)))
);
}
using (Variable.IfNot(selector[featureRange]))
{
weightVar[featureRange].SetTo(Variable.GaussianFromMeanAndVariance(0, 0.0000001).Named("spike"));
}
}
Variable<double> precision = Variable<double>.GammaFromShapeAndScale(1, 99999);
using (Variable.ForEach(dataRange))
{
VariableArray<double> weightSumArray = Variable.Array<double>(featureRange).Named("weightSum");
weightSumArray[featureRange] = (weightVar[featureRange] * inputArr[dataRange][featureRange]).Named("weightSumArray");
// the effect from other nodes - in case of no perturbation
Variable<double> linearEffect =
Variable.Sum(weightSumArray).Named("sum_weightSumArray");
Variable<double> nonLinearEffect = testFunction(linearEffect).Named("nonLinearEffect");
outputArray[dataRange] = Variable.GaussianFromMeanAndPrecision(nonLinearEffect, precision);
}
inputArr.ObservedValue = data;
outputArray.ObservedValue = outputData;
//InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing());
InferenceEngine ie = new InferenceEngine(new ExpectationPropagation());
ie.NumberOfIterations = 50;
//ie.Compiler.GivePriorityTo(typeof(GaussianProductOp_SHG09));
Gaussian[] weightPosterior = ie.Infer<Gaussian[]>(weightVar);
Console.WriteLine("\n*** printing weights posterior...\n");
for (int i = 0; i < weightPosterior.Length; i++)
{
Console.WriteLine("weightPosterior[" + i + "] = " + weightPosterior[i]);
}

Console.WriteLine("\n\n*** printing selector posterior (Bernouli posterior for feature selection)...\n");
Bernoulli[] selectorPosterior = ie.Infer<Bernoulli[]>(selector);
for (int i = 0; i < selectorPosterior.Length; i++)
{
Console.WriteLine("selectorPosterior[" + i + "] = " + selectorPosterior[i]);
}
}

}
}
```

Wednesday, March 25, 2015 1:55 PM

• This turns out to be a scheduling bug.  It will be fixed in the next version.  You can work around it by initializing weightVar:

```            weightVar[featureRange].InitialiseTo(new Gaussian(0, 1));
```

However, even with this fix EP has convergence issues, which is to be expected in a loopy model like this.

Sunday, April 12, 2015 11:24 AM

### All replies

• What error message are you getting?
Wednesday, April 1, 2015 10:41 AM
• This is the error message:

An unhandled exception of type 'System.Exception' occurred in Infer.Runtime.dll

Additional information: x is nan

=======

This is the commandline output before the error:

Compiling model...compilation had 9 warning(s).
 GaussianProductOp.AAverageConditional(vdouble24_B[index0], vdouble23_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
 GaussianProductOp.ProductAverageConditional(vdouble23_B[index0], vdouble22_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimentalwhich is less than the recommended quality band (Preview)
 GaussianProductOp.ProductAverageConditional(vdouble24_B[index0], vdouble23_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
 GaussianProductOp.AAverageConditional(vdouble24_B[index0], vdouble23_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
 GaussianProductOp.ProductAverageConditional(vdouble23_B[index0], vdouble22_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
 GaussianProductOp.ProductAverageConditional(vdouble24_B[index0], vdouble23_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
 GaussianProductOp.BAverageConditional(vdouble24_B[index0], vdouble23_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
 GaussianProductOp.BAverageConditional(vdouble23_B[index0], vdouble22_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
 GaussianProductOp.AAverageConditional(vdouble23_B[index0], vdouble22_F[index0], sum_weightSumArray_uses_F[index0]) has quality band Experimental which is less than the recommended quality band (Preview)
done.

Wednesday, April 1, 2015 5:54 PM
• Try following the advice I gave in the previous thread.
Wednesday, April 1, 2015 9:24 PM
• Thank you Tom.

I have tried all the advices in previous posts (you can see the mentioned line of code in comment).

Can you please let me know if you can run the function "nonlinearFeatureSelection()" successfully when setting "ie.Compiler.GivePriorityTo(typeof(GaussianProductOp_SHG09));"?

(In current code in Main method linearFeatureSelection function is called. But you can easily comment it and uncomment the line of code for running nonlinearFeatureSelection.)

I mean maybe there is a problem with my system (like not having enough memory ...).

I have copied the code to be tested here (Main function should be changed like this....... and  "ie.Compiler.GivePriorityTo(typeof(GaussianProductOp_SHG09));" should be uncomment in the function nonlinearFeatureSelection() ):

```
static void Main(string[] args)
{
//linearFeatureSelection();
nonlinearFeatureSelection();
}

InferenceEngine ie = new InferenceEngine(new ExpectationPropagation());
ie.NumberOfIterations = 50;
ie.Compiler.GivePriorityTo(typeof(GaussianProductOp_SHG09));            // comment should be removed from the above line in nonlinearFeatureSelection() function
```

• Edited by Thursday, April 2, 2015 9:39 PM
Thursday, April 2, 2015 9:34 PM
• This turns out to be a scheduling bug.  It will be fixed in the next version.  You can work around it by initializing weightVar:

```            weightVar[featureRange].InitialiseTo(new Gaussian(0, 1));
```

However, even with this fix EP has convergence issues, which is to be expected in a loopy model like this.

Sunday, April 12, 2015 11:24 AM
• 