none
Job Template Level Activation filter - limit number of jobs per user RRS feed

  • Question

  • The purpose of this job template level activation filter is to limit the number of jobs per user and starts the jobs in the order specified by jobIndex.

    It seems to work but is this a reasonable implementation or could it be done much simpler and/or smarter?

    /*
     *  Windows HPC Server 2012 R2 
     *  Job Template Level Activation Filter
     *  Limits the number of active jobs per user
    */
    
    
    using System;
    using System.Collections;
    using System.Xml;
    using System.IO;
    using System.Reflection;
    
    
    using Microsoft.Hpc.Scheduler;
    using Microsoft.Hpc.Scheduler.Properties;
    using Microsoft.Hpc.Scheduler.AddInFilter.HpcClient;
    
    namespace JobsPerUser
    {
        public class LimitUsersJobs : IActivationFilter, IFilterLifespan  // note: IFilterLifespan is optional
        {
            public TextWriter logFile = null;
            private static int lastSchedulerPass = 0;
            private static int lastJobIndex = 0;
    
            public ActivationFilterResponse FilterActivation(
                Stream jobXml,
                int schedulerPass,
                int jobIndex,
                bool backfill,
                int resourceCount)
            {
                ActivationFilterResponse returnDecision = ActivationFilterResponse.StartJob;
    
                if ((returnDecision = setupLogFile()) != 0)
                {
                    return returnDecision;
                }
    
                try
                {
                    // Load the job file as an XmlDocument.
                    XmlDocument doc = new XmlDocument();
                    doc.Load(jobXml);
    
                    XmlNamespaceManager nsMgr = new XmlNamespaceManager(doc.NameTable);
                    nsMgr.AddNamespace("hpc", "http://schemas.microsoft.com/HPCS2008R2/scheduler/");
    
                    // Find the job node in the XML document.
    
                    XmlNode job = doc.SelectSingleNode("/hpc:Job", nsMgr);
    
                    if (job == null)
                    {
                        throw new Exception("No job in the xml file");
                    }
    
                    // Find the UserName attribute for the job.
    
                    XmlAttributeCollection attrCol = job.Attributes;
                    XmlAttribute userNameAttr = attrCol["UserName"];
    
                    if (userNameAttr != null)
                    {
                        string userName = userNameAttr.Value;
                        long jobCount = 0;
                        // Count number of jobs
                        using (IScheduler scheduler = new Scheduler())
                        {
                            scheduler.Connect("localhost");
                            IFilterCollection filters = scheduler.CreateFilterCollection();
                            filters.Add(FilterOperator.Equal, PropId.Job_UserName, userName);
                            filters.Add(FilterOperator.Equal, PropId.Job_State, JobState.Running);
                            jobCount = scheduler.GetJobList(filters, null).Count;
                            scheduler.Close();
    
                        } // using scheduler
    
                        // Return value
                        if (jobCount > 2)
                        {
                            returnDecision = ActivationFilterResponse.DoNotRunKeepResourcesAllowOtherJobsToSchedule;
                        }
                        else
                        {
                            if ((jobIndex < lastJobIndex) && (schedulerPass > lastSchedulerPass))
                            {
                                returnDecision = ActivationFilterResponse.StartJob;
                                lastSchedulerPass = schedulerPass;
                            }
                            else
                            {
                                returnDecision = ActivationFilterResponse.DoNotRunKeepResourcesAllowOtherJobsToSchedule;
                            }
                        }
    
                        lastJobIndex = jobIndex;
    
    //                    XmlAttribute jobIdAttr = attrCol["Id"];
    //                    if (jobIdAttr != null)
    //                    {
    //                        string jobId = jobIdAttr.Value;
    //                        logFile.WriteLine(userName + " " + jobId + " " + jobCount.ToString() + " " + schedulerPass.ToString() + " " + lastSchedulerPass.ToString() + " " + jobIndex.ToString());
    //                    }
                    }
                }
                catch (IOException e)
                {
                    logFile.WriteLine("Error Loading the XmlFile");
                    logFile.WriteLine(e.ToString());
                }
                catch (Exception e)
                {
                    logFile.WriteLine("Error");
                    logFile.WriteLine(e.ToString());
                }
    
                finally
                {
                    logFile.Close();
                }
    
                return returnDecision;
            }
            public void OnFilterLoad()
            {
                // setup code here
            }
    
            public void OnFilterUnload()
            {
                // teardown code here
            }
    
            public void RevertActivation(Stream jobXml, int schedulerPass, int jobIndex, bool backfill, int resourceCount)
            {
                //
            }
    
            private ActivationFilterResponse setupLogFile()
            {
                try
                {
                    string assemblyPathInclusive = Assembly.GetExecutingAssembly().Location;
                    string assemblyPath = Path.GetDirectoryName(assemblyPathInclusive);
                    String logFileName = Path.Combine(assemblyPath, "ActivationFilterLog.log");
                    logFile = new StreamWriter(logFileName, true);
                    return ActivationFilterResponse.StartJob;
                }
                catch (Exception ex)
                {
                    return ActivationFilterResponse.StartJob;
                }
            }
    
    
    
        } // class
    } // namespace
    

    Thursday, August 6, 2015 1:34 PM

Answers

  • Hi Thomas,

      You are using the right approach as I see. The issue I see is that there might be jobs sitting in the QUEUE while there are still idle resources. Depending on your scheduling mode (Queued and Balanced), you can also try below approach leveraging "Job Submission Filter":

    - In the job submissions, check the job username (Or job owner)

    - Get the number jobs that already in the QUEUE with the same job user (or job owner)

    - If the number is jobs is less than N, put the job priority as "High", otherwise "Normal" or "Low"

    In the "job activation Filter":

    - when a new job started with job user (or job owner)

    - Get the list of jobs belong to the same job user (or job owner), sorted by submission time

    - Make sure the first N jobs are with job priority "High"

    This case, you have two job queued by priority. There will be at most N jobs for a job user (or Job owner) are in the high priority job queue which will be scheduled first before the normal queue. Within one queue, jobs are scheduled according to their submission time.


    Qiufang Shi

    • Marked as answer by Thomas Kofoed Monday, August 10, 2015 8:50 AM
    Monday, August 10, 2015 4:16 AM

All replies

  • Hi Thomas,

      You are using the right approach as I see. The issue I see is that there might be jobs sitting in the QUEUE while there are still idle resources. Depending on your scheduling mode (Queued and Balanced), you can also try below approach leveraging "Job Submission Filter":

    - In the job submissions, check the job username (Or job owner)

    - Get the number jobs that already in the QUEUE with the same job user (or job owner)

    - If the number is jobs is less than N, put the job priority as "High", otherwise "Normal" or "Low"

    In the "job activation Filter":

    - when a new job started with job user (or job owner)

    - Get the list of jobs belong to the same job user (or job owner), sorted by submission time

    - Make sure the first N jobs are with job priority "High"

    This case, you have two job queued by priority. There will be at most N jobs for a job user (or Job owner) are in the high priority job queue which will be scheduled first before the normal queue. Within one queue, jobs are scheduled according to their submission time.


    Qiufang Shi

    • Marked as answer by Thomas Kofoed Monday, August 10, 2015 8:50 AM
    Monday, August 10, 2015 4:16 AM
  • Many thanks for your feedback and suggestion of an alternative using the job priority!

    /Thomas 

    Monday, August 10, 2015 8:53 AM