locked
Out of memory Exception RRS feed

  • Question

  • Hi

     

    I am getting an out of memory exception when trying to infer on a discrete bayes net. I do not provide any evidence, I want to find the likelihood of a node.

    Eg. P(Body = SUV).

    I have 6 nodes in my bayes net. And the size of the discrete distributions are {416,25,50,90,50}.

    Is there anyway in which I can avoid the exception.

    Thanks

    Rohit

     

     

    Monday, July 4, 2011 6:38 AM

All replies

  • Hi Rohit

    Out of memory exceptions can occur when the entire factor graph with all forward and backward messages is in memory at the same time. There are ways to split up the factor graph to avoid this. My suggestions is that you work with a small example initially which fits in memory. When you have got that stable, then work to get it split up. There are several posts on this (see http://social.microsoft.com/Forums/en-US/infer.net/thread/94bee5c7-056c-4933-81d0-cbc80bd4d888 for example), and many references in the user guide (see the FAQ, and the various sections on shared variables). Let me know if you need help with this, but, if possible, please provide a simplified version of your model and data to work with.

    John

    Monday, July 4, 2011 9:24 AM
    Owner
  •  

    Disambiguation: There is a node called "Model" in my Bayes Network.

    ---

    Bayes Network Structure:

    Myear - no parents

    Model - 1 parent (Myear)

    Body - 2 parents (Model, Myear)

    Make - 1 parent(Model)

    Price - 2 parents (Model, Myear)

    Mileage - 1 parent (Myear)

    ---

    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Text;
    using MicrosoftResearch.Infer.Models;
    using MicrosoftResearch.Infer;
    using MicrosoftResearch.Infer.Maths;
    using MicrosoftResearch.Infer.Distributions;
    using MicrosoftResearch.Infer.Factors;

    namespace BN_2
    {
        using DirichletArray = DistributionRefArray<Dirichlet, Vector>;
        using DirichletArray2D = DistributionRefArray2D<Dirichlet, Vector>;
        [Serializable]
        class CarBayesNet
        {
            // Model variables
            [NonSerialized]
            public Variable<int> NumCases = Variable.New<int>();
            [NonSerialized]
            public VariableArray<int> CarModel;
            [NonSerialized]
            public VariableArray<int> CarMyear;
            [NonSerialized]
            public VariableArray<int> CarBody;
            [NonSerialized]
            public VariableArray<int> CarMake;
            [NonSerialized]
            public VariableArray<int> CarPrice;
            [NonSerialized]
            public VariableArray<int> CarMileage;

            [NonSerialized]
            public VariableArray<int>[] allCarVars;

            // Probability tables and conditional probability tables
            [NonSerialized]
            public VariableArray<Vector> CPT_Model;
            [NonSerialized]
            public Variable<Vector> CPT_Myear;
            [NonSerialized]
            public VariableArray2D<Vector> CPT_Body;
            [NonSerialized]
            public VariableArray<Vector> CPT_Make;
            [NonSerialized]
            public VariableArray2D<Vector> CPT_Price;
            [NonSerialized]
            public VariableArray<Vector> CPT_Mileage;

            [NonSerialized]
            public IVariable[] allCPTs;

            // Prior distributions for the probability tables - make these variable so we can
            // change at run-time without model recompilation
            [NonSerialized]
            public Variable<DirichletArray> CPT_Model_Prior = null;
            [NonSerialized]
            public Variable<Dirichlet> CPT_Myear_Prior = null;
            [NonSerialized]
            public Variable<DirichletArray2D> CPT_Body_Prior = null;
            [NonSerialized]
            public Variable<DirichletArray> CPT_Make_Prior = null;
            [NonSerialized]
            public Variable<DirichletArray2D> CPT_Price_Prior = null;
            [NonSerialized]
            public Variable<DirichletArray> CPT_Mileage_Prior = null;

            [NonSerialized]
            public IVariable[] allCPT_Prior;

            // Inferred posterior distributions over the probability tables
            public DirichletArray CPT_Model_Posterior = null;
            public Dirichlet CPT_Myear_Posterior = null;
            public DirichletArray2D CPT_Body_Posterior = null;
            public DirichletArray CPT_Make_Posterior = null;
            public DirichletArray2D CPT_Price_Posterior = null;
            public DirichletArray CPT_Mileage_Posterior = null;

            public List<int> savedNumObjects;

            // Inference engine
            [NonSerialized]
            //public InferenceEngine InfEngine = new InferenceEngine() { ShowProgress = false };
            public InferenceEngine InfEngine = new InferenceEngine() { ShowProgress = false, ShowTimings = false, NumberOfIterations = 10 };

            public void CreateModel(List<int> numObjects)
            {


                NumCases = Variable.New<int>(); // Set this at run time

                savedNumObjects = numObjects; // so that this can be serialized
                // The ranges and value ranges
                Range n = new Range(NumCases);
                Range RangeModel = new Range(numObjects[(int)CarAttr.Model]);
                Range RangeMyear = new Range(numObjects[(int)CarAttr.Myear]);
                Range RangeBody = new Range(numObjects[(int)CarAttr.Body]);
                Range RangeMake = new Range(numObjects[(int)CarAttr.Make]);
                Range RangePrice = new Range(numObjects[(int)CarAttr.Price]);
                Range RangeMileage = new Range(numObjects[(int)CarAttr.Mileage]);
               


                // Define the priors as variables - these will be set (observed) at run time
                CPT_Model_Prior = Variable.New<DirichletArray>();
                CPT_Myear_Prior = Variable.New<Dirichlet>();
                CPT_Body_Prior = Variable.New<DirichletArray2D>();
                CPT_Make_Prior = Variable.New<DirichletArray>();
                CPT_Price_Prior = Variable.New<DirichletArray2D>();
                CPT_Mileage_Prior = Variable.New<DirichletArray>();
              

                allCPT_Prior = new IVariable[] { CPT_Model_Prior, CPT_Myear_Prior, CPT_Body_Prior,
                    CPT_Make_Prior, CPT_Price_Prior, CPT_Mileage_Prior };

                // Probability table and conditional probability table variables (parameters)
                CPT_Model = Variable.Array<Vector>(RangeMyear);
                CPT_Model.SetTo(Variable<Vector[]>.Random(CPT_Model_Prior));

                CPT_Myear = Variable<Vector>.Random(CPT_Myear_Prior);
               

                CPT_Body = Variable.Array<Vector>(RangeModel, RangeMyear);
                CPT_Body.SetTo(Variable<Vector[,]>.Random(CPT_Body_Prior));

                CPT_Make = Variable.Array<Vector>(RangeModel);
                CPT_Make.SetTo(Variable<Vector[]>.Random(CPT_Make_Prior));

                CPT_Price = Variable.Array<Vector>(RangeModel, RangeMyear);
                CPT_Price.SetTo(Variable<Vector[,]>.Random(CPT_Price_Prior));

                CPT_Mileage = Variable.Array<Vector>(RangeMyear);
                CPT_Mileage.SetTo(Variable<Vector[]>.Random(CPT_Mileage_Prior));

              
                // Set the value ranges for the probability tables
                CPT_Model.SetValueRange(RangeModel);
                CPT_Myear.SetValueRange(RangeMyear);
                CPT_Body.SetValueRange(RangeBody);
                CPT_Make.SetValueRange(RangeMake);
                CPT_Price.SetValueRange(RangePrice);
                CPT_Mileage.SetValueRange(RangeMileage);
              

                // Describe the structure
                //CarModel = Variable.Array<int>(n);
                //CarModel[n] = Variable.Discrete(CPT_Model).ForEach(n);

              

                CarMyear = Variable.Array<int>(n);
                CarMyear[n] = Variable.Discrete(CPT_Myear).ForEach(n);
                //CarMyear = AddChildFromOneParent(CarModel, CPT_Myear);
                CarModel = AddChildFromOneParent(CarMyear, CPT_Model);
                CarBody = AddChildFromTwoParents(CarModel, CarMyear, CPT_Body);

                CarMake = AddChildFromOneParent(CarModel, CPT_Make);

                CarPrice = AddChildFromTwoParents(CarModel, CarMyear, CPT_Price);

                CarMileage = AddChildFromOneParent(CarMyear, CPT_Mileage);

                allCarVars = new VariableArray<int>[] { CarModel, CarMyear, CarBody, CarMake, CarPrice, CarMileage };
            }

            // Infer the probability tables and conditional probability tables
            public void InferParameters(int[][] carValues)
            {
                // Set the observations:
                NumCases.ObservedValue = carValues[0].Length;
                CarModel.ObservedValue = carValues[(int)CarAttr.Model];
                CarMyear.ObservedValue = carValues[(int)CarAttr.Myear];
                CarBody.ObservedValue = carValues[(int)CarAttr.Body];
                CarMake.ObservedValue = carValues[(int)CarAttr.Make];
                CarPrice.ObservedValue = carValues[(int)CarAttr.Price];
                CarMileage.ObservedValue = carValues[(int)CarAttr.Mileage];
               


                // Set the uniform priors for the probability tables
                int numModels = CarModel.GetValueRange().SizeAsInt;
                int numMyear = CarMyear.GetValueRange().SizeAsInt;
                int numBody = CarBody.GetValueRange().SizeAsInt;
                int numMake = CarMake.GetValueRange().SizeAsInt;
                int numPrice = CarPrice.GetValueRange().SizeAsInt;
                int numMileage = CarMileage.GetValueRange().SizeAsInt;


                CPT_Model_Prior.ObservedValue = GetUniformPrior(numMyear, numModels);
                CPT_Myear_Prior.ObservedValue = Dirichlet.Uniform(numMyear);
                CPT_Body_Prior.ObservedValue = GetUniformPrior(numModels, numMyear, numBody);
                CPT_Make_Prior.ObservedValue = GetUniformPrior(numModels, numMake);
                CPT_Price_Prior.ObservedValue = GetUniformPrior(numModels, numMyear, numPrice);
                CPT_Mileage_Prior.ObservedValue = GetUniformPrior(numMyear, numMileage);
               

                // Run the inference
                InfEngine.InferAll(CPT_Model, CPT_Myear, CPT_Body, CPT_Make, CPT_Price, CPT_Mileage);
                CPT_Model_Posterior = InfEngine.Infer<DirichletArray>(CPT_Model);
                CPT_Myear_Posterior = InfEngine.Infer<Dirichlet>(CPT_Myear);
                CPT_Body_Posterior = InfEngine.Infer<DirichletArray2D>(CPT_Body);
                CPT_Make_Posterior = InfEngine.Infer<DirichletArray>(CPT_Make);
                CPT_Price_Posterior = InfEngine.Infer<DirichletArray2D>(CPT_Price);
                CPT_Mileage_Posterior = InfEngine.Infer<DirichletArray>(CPT_Mileage);
               

            }

            //done so far

            private DirichletArray GetUniformPrior(int numParent1, int numRange)
            {
                Dirichlet[] dMil = new Dirichlet[numParent1];
                for (int i = 0; i < numParent1; i++)
                {
                    dMil[i] = Dirichlet.Uniform(numRange);
                }

                return (DirichletArray)Distribution<Vector>.Array(dMil);
            }

            private DirichletArray2D GetUniformPrior(int numParent1, int numParent2, int numRange)
            {
                Dirichlet[,] dMil = new Dirichlet[numParent1, numParent2];
                for (int i = 0; i < numParent1; i++)
                {
                    for (int j = 0; j < numParent2; j++)
                    {
                        dMil[i, j] = Dirichlet.Uniform(numRange);
                    }
                }
                return (DirichletArray2D)Distribution<Vector>.Array(dMil);
            }

            // Query the distribution over car type conditioned on each of a set of model types
            public Discrete[] QueryCarTypesFromModelTypes(int[] models)
            {
                ClearAllObservedValues();
                NumCases.ObservedValue = models.Length;
                CarModel.ObservedValue = models;

                MirrorPosteriorToObserved2();

                // Run the inference
                return InfEngine.Infer<Discrete[]>(CarMake);
            }

            //Rohit Aim of the method: Given a row, predict some value.
            public Discrete[] QueryModel(int[] makes, int[] body)
            {
                ClearAllObservedValues();
                NumCases.ObservedValue = makes.Length;
                CarMake.ObservedValue = makes;
                CarBody.ObservedValue = body;

                MirrorPosteriorToObserved2();

                //Run the inference
                return InfEngine.Infer<Discrete[]>(CarModel);

            }

            /// <summary>
            /// Copies the learnt posterior values to the observed values of prior and the CPT
            /// </summary>
            public void MirrorPosteriorToObserved()
            {
                CPT_Model_Prior.ObservedValue = CPT_Model_Posterior;
                CPT_Myear_Prior.ObservedValue = CPT_Myear_Posterior;
                CPT_Body_Prior.ObservedValue = CPT_Body_Posterior;
                CPT_Make_Prior.ObservedValue = CPT_Make_Posterior;
                CPT_Price_Prior.ObservedValue = CPT_Price_Posterior;
                CPT_Mileage_Prior.ObservedValue = CPT_Mileage_Posterior;
               

            }

            // Model code for adding a child from a single parent
            public static VariableArray<int> AddChildFromOneParent(
              VariableArray<int> parent, VariableArray<Vector> cpt)
            {
                var d = parent.Range;
                // data range
                var child = Variable.Array<int>(d);
                using (Variable.ForEach(d))
                using (Variable.Switch(parent[d]))
                    child[d] = Variable.Discrete(cpt[parent[d]]);
                return child;
            }

            // Model code for adding a child from two parents
            public static VariableArray<int> AddChildFromTwoParents(
              VariableArray<int> parent1, VariableArray<int> parent2, VariableArray2D<Vector> cpt)
            {
                var d = parent1.Range;
                // data range
                var child = Variable.Array<int>(d);
                using (Variable.ForEach(d))
                using (Variable.Switch(parent1[d]))
                using (Variable.Switch(parent2[d]))
                    child[d] = Variable.Discrete(cpt[parent1[d], parent2[d]]);
                return child;
            }

            public void ClearAllObservedValues()
            {
                foreach (var item in allCarVars)
                {
                    item.ClearObservedValue();
                }
            }

            public void MirrorPosteriorToObserved2()
            {
                CPT_Model.ObservedValue = GetMean(CPT_Model_Posterior.ToArray());
                CPT_Myear.ObservedValue = GetMean(CPT_Myear_Posterior);
                CPT_Body.ObservedValue = GetMean(CPT_Body_Posterior.ToArray());
                CPT_Make.ObservedValue = GetMean(CPT_Make_Posterior.ToArray());
                CPT_Price.ObservedValue = GetMean(CPT_Price_Posterior.ToArray());
                CPT_Mileage.ObservedValue = GetMean(CPT_Mileage_Posterior.ToArray());
               

            }

            public Vector GetMean(Dirichlet dist)
            {
                return dist.GetMean();
            }
            public Vector[] GetMean(Dirichlet[] dists)
            {
                Vector[] result = new Vector[dists.Length];
                for (int i = 0; i < result.Length; i++)
                {
                    result[i] = dists[i].GetMean();
                }
                return result;
            }
            public Vector[,] GetMean(Dirichlet[,] dists)
            {
                int rows = dists.GetLength(0);
                int cols = dists.GetLength(1);
                Vector[,] result = new Vector[rows, cols];
                for (int i = 0; i < rows; i++)
                {
                    for (int j = 0; j < cols; j++)
                    {
                        result[i, j] = dists[i, j].GetMean();
                    }
                }
                return result;
            }

        }
    }

    Monday, July 4, 2011 10:09 AM
  • Hi Rohit

    Let's wait until your model is fixed (i.e. jagged rather than 2-D array) before addressing scalability (also, it would be better to work with an even more pruned down model than this). In the meantime, please read through the SharedVariable sections in the user guide. Also, let me know what is the minimum number of examples to cause the OOM exception, and the total number of examples you have.

    John

    Monday, July 4, 2011 1:46 PM
    Owner
  • Hi John,

    I am actually trying to fix other things, before moving on to the aforementioned problem. 

    It would be great if my inference was faster. I am trying to find if I can do that. I was reading the user guide on the steps for improving the speed.

    When I make showtimings to true. I notice that everytime I call engine.infer there is a compilation happening. Is this normal or is there a way of avoiding this. I change the observed values everytime I call engine.infer

     

    Rohit

     

     

     

    Monday, July 11, 2011 6:05 AM
  • The main reason it is likely to be recompiling each time is if you are changing which variables are observed (which changes the model and causes a recompile) rather than what the observations are. The typical scenario is when you switch between inferring parameters and prediction. The easiest way to get around this is to create two different instances of your model:

     

    CarBayesNet bayesNetTrain = new CarBayesNet();
    CarBayesNet bayesNetPredict = new CarBayesNet

    ();
    bayesNetTrain.CreateModel(numModels, numTypes, numPowerTrains, numEngines);
    bayesNetPredict.CreateModel(numModels, numTypes, numPowerTrains, numEngines);

    Let me know if this does the trick - if not we will need to look deeper.

    Once you have got the compilation issue dealt with, you can also take a look at the posts in http://social.microsoft.com/Forums/en-US/infer.net/thread/f9c91b9b-7583-4069-a4aa-903618754657 which touch on performance.

    As regards the memory issue, before going to shared variables, you could look at sparsifying the messages. For example:

    CPT_E_Prior.SetSparsity(

    Sparsity.Sparse);
    CPT_T_Prior.SetSparsity(
    Sparsity

    .Sparse);

    and

     

    for (inti = 0; i < numModels; i++)
    {
        cpt_E_Prior[i] =
    new Dirichlet[numPowerTrains];
       
    for (intj = 0; j < numPowerTrains; j++)
            cpt_E_Prior[i][j] =
    Dirichlet.Uniform(numEngines, Sparsity.Sparse);
    }
    for (inti = 0; i < numEngines; i++)
        cpt_T_Prior[i] =
    Dirichlet.Uniform(numTypes, Sparsity

    .Sparse);

    Let me know how that goes.

    John

    Monday, July 11, 2011 1:15 PM
    Owner
  • Hi John,

     

    Both Creation of multiple networks trick and using sparse messages did not work. 

     

    Rohit

    Tuesday, July 12, 2011 10:50 AM
  • Hi Rohit

    Apologies for the delay - some work deadlines.

    The model will only recompile if you are changing the observation structure - i.e. if you are observing one set of variables in one query of the model, and then another set of variables in the next query, and this will be solved if you have two different instances. Of course each instance will compile the first time it is called but thereafter will not compile. If you are not seeing this, then I need more information to understand your situation.

    This brings me to the second issue. I started putting together a 'shared variable' form of your model so that only part of the model is in memory at any one time. However, before I went further, I wanted to ask you whether, for training, your model is full observed, i.e. do you always have model, powertrain, engine type, and car type for training? If so,  then the conditional probability tables can be learnt independently - just create a simple model for each of these, and this will probably solve your memory issue (if not it is a very small change to use shared variables on these simple models). Your prediction model can then combine all the cpts (or cpt priors if you want to retain the cpt uncertainties) into a single model. Is that the case?

    John

    Tuesday, July 19, 2011 2:33 PM
    Owner
  • Hi John,

    My training data is fully observed. It has always has a value for all the variables.

     

    Rohit

     

    Tuesday, July 19, 2011 8:12 PM
  • In that case you can train each PT/CPT seperately. So for example, the engine type CPT would look as follows:

      public class EngineTypeBayesNet

      {

        // Model variables

        public Variable<int> NumCases = Variable.New<int>();

        public VariableArray<int> CarModel;

        public VariableArray<int> PowerTrain;

        public VariableArray<int> EngineType;

        public VariableArray<VariableArray<Vector>, Vector[][]> CPT_E;

     

        // Prior distribution for the probability table

        public Variable<DirichletArrayArray> CPT_E_Prior = null;

     

        // Inferred posterior distributions over the probability table

        public DirichletArrayArray CPT_E_Posterior = null;

     

        // Inference engine

        public InferenceEngine InfEngine = new InferenceEngine() { ShowProgress = false };

     

        public void CreateModel(int numModels, int numPowerTrains, int numEngines)

        {

          NumCases = Variable.New<int>().Named("numCases"); // Set this at run time

     

          // The ranges and value ranges

          Range n = new Range(NumCases).Named("n");

          Range m = new Range(numModels).Named("m");

          Range p = new Range(numPowerTrains).Named("p");

          Range e = new Range(numEngines).Named("e");

     

          CPT_E_Prior = Variable.New<DirichletArrayArray>().Named("etprior");

          CPT_E = Variable.Array(Variable.Array<Vector>(p), m).Named("et");

          CPT_E.SetTo(Variable<Vector[][]>.Random(CPT_E_Prior));

          CPT_E.SetValueRange(e);

     

          // Describe the structure

          CarModel = Variable.Array<int>(n).Named("model");

          PowerTrain = Variable.Array<int>(n).Named("powertrain");

          CarModel.SetValueRange(m);

          PowerTrain.SetValueRange(p);

     

          EngineType = AddChildFromTwoParents(CarModel, PowerTrain, CPT_E, "enginetype");

        }

     

        // Infer the probability tables and conditional probability tables

     

        public void InferParameters(int[] modelData, int[] ptData, int[] engineData)

        {

          // Set the observations:

          NumCases.ObservedValue = modelData.Length;

          CarModel.ObservedValue = modelData;

          PowerTrain.ObservedValue = ptData;

          EngineType.ObservedValue = engineData;

     

          // Set the uniform priors for the probability tables

          int numModels = CarModel.GetValueRange().SizeAsInt;

          int numPowerTrains = PowerTrain.GetValueRange().SizeAsInt;

          int numEngines = EngineType.GetValueRange().SizeAsInt;

     

          Dirichlet[][] cpt_E_Prior = new Dirichlet[numModels][];

          for (int i = 0; i < numModels; i++)

          {

            cpt_E_Prior[i] = new Dirichlet[numPowerTrains];

            for (int j = 0; j < numPowerTrains; j++)

     

              cpt_E_Prior[i][j] = Dirichlet.Uniform(numEngines, Sparsity.Sparse);

          }

     

          CPT_E_Prior.ObservedValue = (DirichletArrayArray)Distribution<Vector>.Array(cpt_E_Prior);

     

          // Run the inference

          InfEngine.InferAll(CPT_E);

          CPT_E_Posterior = InfEngine.Infer<DirichletArrayArray>(CPT_E);

        }

     

        // Model code for adding a child from two parents

        public static VariableArray<int> AddChildFromTwoParents(

          VariableArray<int> parent1, VariableArray<int> parent2,

          VariableArray<VariableArray<Vector>, Vector[][]> cpt, string name)

        {

          var d = parent1.Range;

          // data range

          var child = Variable.Array<int>(d).Named(name);

          using (Variable.ForEach(d))

          using (Variable.Switch(parent1[d]))

          using (Variable.Switch(parent2[d]))

            child[d] = Variable.Discrete(cpt[parent1[d]][parent2[d]]);

          return child;

        }

      }

     

    And it would be called as follows:

          EngineTypeBayesNet engineNet = new EngineTypeBayesNet();

          engineNet.CreateModel(numModels, numPowerTrains, numEngines);

          engineNet.InferParameters(modelData, ptData, engineData);

     

    All the other training models are simpler than this. If you are still having memory issues this can be converted to a shared variable model with a few lines of code - let me know. Your prediction model will be as before, using the posteriors from each of the individual models as the priors for the prediction model. 

    Wednesday, July 20, 2011 12:55 PM
    Owner