Hi,
Using Infer.NET, I've tried and tested the Gaussian Mixture Model with distributions on mean vectors (centroids), mixture weights and covariance matrices and everything works fine (with datasets obtained from
here). But for my research, as I have to deal with large datasets with very high dimensions (more than 10000 data points with each datapoint of dimension 3780), to reduce the number of free parameters, I would
like to tie/share the covariance matrices and fix it to identity matrix (I'm guessing that this will approach to Bayesian-Kmeans).
I've written the code for this and it does work for some datasets but for other datasets, it gives the following error (I've also attached my code below).
Compiling model...done.
Iterating:
........Unhandled Exception: MicrosoftResearch.Infer.Maths.AllZeroException: The model has zero probability
at MicrosoftResearch.Infer.Distributions.Discrete.Normalize() in C:\infernetBuilds\12-10-2011_14-16\Runtime\Distributions\Discrete.cs:line 541
at MicrosoftResearch.Infer.Distributions.Discrete.SetToProduct(Discrete a, Discrete b) in C:\infernetBuilds\12-10-2011_14-16\Runtime\Distributions\Discrete.cs:line 366
at MicrosoftResearch.Infer.Factors.VariableVmpOp.MarginalAverageLogarithm[T](T use, T def, T result) in C:\infernetBuilds\12-10-2011_14-16\Runtime\Factors\VariableOp.cs:line 285
at MicrosoftResearch.Infer.Models.User.Model_VMP.Changed_numberOfIterationsDecreased_x_vIDistributionLtint__Gt0(Int32 numberOfIterations)
at MicrosoftResearch.Infer.Models.User.Model_VMP.Execute(Int32 numberOfIterations, Boolean initialise)
at MicrosoftResearch.Infer.Models.User.Model_VMP.Execute(Int32 numberOfIterations)
at MicrosoftResearch.Infer.InferenceEngine.Execute(IGeneratedAlgorithm ca) in C:\infernetBuilds\12-10-2011_14-16\Compiler\Infer\InferenceEngine.cs:line 519
at MicrosoftResearch.Infer.InferenceEngine.InferAll(Boolean inferOnlySpecifiedVars, IVariable var) in C:\infernetBuilds\12-10-2011_14-16\Compiler\Infer\InferenceEngine.cs:line 494
at MicrosoftResearch.Infer.InferenceEngine.Infer[TReturn](IVariable var) in C:\infernetBuilds\12-10-2011_14-16\Compiler\Infer\InferenceEngine.cs:line 290
Code for Shared identity Covariance GMM
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using MicrosoftResearch.Infer.Models;
using MicrosoftResearch.Infer;
using MicrosoftResearch.Infer.Distributions;
using MicrosoftResearch.Infer.Maths;
using MicrosoftResearch.Infer.Utils;
namespace GMM_CS
{
class gmm_cs
{
static void Main(string[] args)
{
////////// Read in data /////////////////
BinaryReader binReader = new BinaryReader(File.Open(@"dataset_clustering.bin", FileMode.Open));
int ndata = binReader.ReadInt32();
int dim = binReader.ReadInt32();
int nclusters = binReader.ReadInt32();
int numiters = binReader.ReadInt32();
int burnin = binReader.ReadInt32();
int thin = binReader.ReadInt32();
double[,] dataset = new double[ndata, dim];
double eta0, alpha_temp;
double[] alpha = new double[nclusters];
Vector m0 = Vector.Zero(dim);
PositiveDefiniteMatrix B0 = PositiveDefiniteMatrix.Identity(dim);
alpha_temp = binReader.ReadDouble();
for (int i = 0; i < nclusters; i++)
alpha[i] = alpha_temp;
eta0 = binReader.ReadDouble();
for (int j = 0; j < dim; j++)
m0[j] = binReader.ReadDouble();
for (int i = 0; i < dim; i++)
for (int j = 0; j < dim; j++)
B0[i, j] = binReader.ReadDouble();
for (int j = 0; j < dim; j++)
for (int i = 0; i < ndata; i++)
dataset[i, j] = binReader.ReadDouble();
binReader.Close();
///////// Specify Distributions //////
// Define a range for the number of mixture components
Range k = new Range(nclusters).Named("k");
// Mixture component means
VariableArray<Vector> means = Variable.Array<Vector>(k).Named("means");
means[k] = Variable.VectorGaussianFromMeanAndPrecision(m0, B0).ForEach(k);
// Identity covariance matrix for all components (observed)
Variable<PositiveDefiniteMatrix> prec = PositiveDefiniteMatrix.Identity(dim);
// Mixture weights
Variable<Vector> weights = Variable.Dirichlet(k, alpha).Named("weights");
// Create a variable array which will hold the data
Range n = new Range(ndata).Named("n");
VariableArray<Vector> data = Variable.Array<Vector>(n).Named("x");
// Create latent indicator variable for each data point
VariableArray<int> z = Variable.Array<int>(n).Named("z");
// The mixture of Gaussians model
using (Variable.ForEach(n))
{
z[n] = Variable.Discrete(weights);
using (Variable.Switch(z[n]))
{
data[n] = Variable.VectorGaussianFromMeanAndPrecision(means[z[n]], prec);
}
}
double[] cur_row = new double[dim];
data.ObservedValue = new Vector[ndata];
for (int i = 0; i < ndata; i++)
{
for (int j = 0; j < dim; j++)
{
cur_row[j] = dataset[i, j];
}
data.ObservedValue[i] = Vector.FromArray(cur_row);
}
// Initialise messages randomly so as to break symmetry
Discrete[] zinit = new Discrete[n.SizeAsInt];
for (int i = 0; i < zinit.Length; i++)
zinit[i] = Discrete.PointMass(Rand.Int(k.SizeAsInt), k.SizeAsInt);
z.InitialiseTo(Distribution<int>.Array(zinit));
// The inference
InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing());
//GibbsSampling gs = new GibbsSampling();
//gs.BurnIn = burnin;
//gs.Thin = thin;
//InferenceEngine ie = new InferenceEngine(gs);
//ie.NumberOfIterations = numiters;
////////////////////////////// Obtaining Output & Writing into binary file /////////////
BinaryWriter binWriter = new BinaryWriter(File.Open("output_gmm.bin", FileMode.Create));
//////////////// Mixing Weights Output ////////////////////
Vector mweights = ie.Infer<Dirichlet>(weights).GetMean();
double[] mix_weights = mweights.ToArray();
for (int i = 0; i < mix_weights.Length; i++)
binWriter.Write(mix_weights[i]);
////////////// Centroids Output ////////////////////
VectorGaussian[] vvg = ie.Infer<VectorGaussian[]>(means);
Vector cur_centroid;
for (int i = 0; i < nclusters; i++)
{
cur_centroid = vvg[i].GetMean();
for (int j = 0; j < dim; j++)
binWriter.Write(cur_centroid[j]);
}
////////////////// Precision Matrices Output ////////////////////
//Wishart[] precs_pre = ie.Infer<Wishart[]>(precs);
//PositiveDefiniteMatrix cur_precision;
//for (int i = 0; i < nclusters; i++)
//{
// cur_precision = precs_pre[i].GetMean();
// for (int ii = 0; ii < dim; ii++)
// for (int jj = 0; jj < dim; jj++)
// binWriter.Write(cur_precision[ii, jj]);
//}
binWriter.Close();
Console.WriteLine("End of Inference.");
//Console.ReadLine();
}
}
}
I'm not really sure what's causing the error.
Thanks.
