Convergence problem in simple regression with learning noise

• Question

• 

  

  0

  Sign in to vote

  Hi,

  I am trying to run a simple regression model based on:

  if a make the noise (noise=0.1) a fixed value it works great, also making noise with a gamma prior (see code below) works great for very small data set (intNumObservaciones=10) but if I increase a little bit the dataset (intNumObservaciones=25) I have convergence problems-> System.Exception: 'Not converging for n=0,x=NaN'. I have tried differente setting of the Gamma without luck also other algorithm but with out luck.

  Any help with that error?

  Anyhow basically I wanted not to have a fixed noise since I wanted a probability distribution for y when I predicted a new value instead of a point estimated. Maybe there is another option to have y predicted as a distribution?.

  Thanks,

  Jaime

  CODE:

  // This is based on: https://github.com/usptact/Infer.NET-BayesianRegression
              // Create synthetic data Y=B+10X1+20X2+error (uniform(0,20)), X1 and X2 Uniform[1,10]
              Vector[] xdata;
              double[] distress;
              List<double> lstEcuacion = new List<double>();
              lstEcuacion.Add(10);
              lstEcuacion.Add(20);
              Int32 intErrorMax = 20;
              Int32 intNumObservaciones = 25;
              xdata = new Vector[intNumObservaciones];
              distress = new double[intNumObservaciones];
              Int32 intNumFactores = lstEcuacion.Count;
              Random rnd = new Random();
              for (Int32 intI = 0; intI < intNumObservaciones; intI++)
              {
                  double dblYtmp = 0;
                  double[] dblXTmp = new double[lstEcuacion.Count + 1];
                  dblXTmp[0] = 1;
                  for (Int32 intFactor = 0; intFactor < lstEcuacion.Count; intFactor++)
                  {
                      Int32 intFlag = rnd.Next(1, 10);
                      dblYtmp = dblYtmp + lstEcuacion[intFactor] * intFlag;
                      dblXTmp[intFactor + 1] = intFlag;
                  }
                  dblYtmp = dblYtmp + rnd.Next(0, intErrorMax);
                  distress[intI] = dblYtmp;
                  xdata[intI] = Vector.FromArray(dblXTmp.ToArray());
              }
  
              // define a prior distribution and attach that to "w" random variable
              VectorGaussian wPrior = new VectorGaussian(Vector.Zero(intNumFactores + 1), PositiveDefiniteMatrix.Identity(intNumFactores + 1));
              Variable<Vector> w = Variable.Random(wPrior);
  
              //Noise distribution mean of a Gamma distribution is shape * scale, and its variance is shape * scale * scale
              double shape = 1000;
              double scale = 0.0001;
              Gamma noiseDist = new Gamma(shape, scale);
              Variable<double> noise = Variable.Random(noiseDist);
  
              // set features "x" and observations "y" as observed in the model
              VariableArray<double> y = Variable.Observed(distress);
              Range n = y.Range;
              VariableArray<Vector> x = Variable.Observed(xdata, n);
  
              // define "y" statistically: Gaussian RV array. Mean is defined by dot-product of param vector "w" and the feature vector x[n]
              y[n] = Variable.GaussianFromMeanAndVariance(Variable.InnerProduct(w, x[n]), noise);
  
              // Training
              InferenceEngine engine = new InferenceEngine();
              engine.ModelName = "Regression";
              engine.Algorithm = new ExpectationPropagation();
              // engine.Algorithm = new VariationalMessagePassing();
              //engine.Algorithm = new GibbsSampling();
  
              // infer "w" posterior as a distribution
              VectorGaussian wPosterior = engine.Infer<VectorGaussian>(w);
              Gamma noisePosterior = engine.Infer<Gamma>(noise);
              Console.WriteLine("Distribution over w = \n" + wPosterior);
              Console.WriteLine("Distribution over noise = \n" + noisePosterior);

  Sunday, July 8, 2018 9:55 AM