Here is a C# Nunit test that reproduces the problem. I ran this on an HPC cluster with 2 compute nodes, each with 4 cores and 1 socket.
I submit a job with 1 slow task, in order to make an uneven number of nodes available.
I then submit a job with 4 tasks - each task requires 2 cores. I then check that all of the tasks only executed on one node. This always fails with "Should have only run on 1 core but ran on 2 cores (MyNodeA,MyNodeB)" One of the tasks gets split across
2 nodes.
[TestFixture]
public class HpcTests
{
[Test]
public void Should_run_each_task_on_only_one_node()
{
const string hpcServer = "HOSTNAME_OF_HPC_SERVER_GOES_HERE";
const string hpcUsername = "USERNAME_GOES_HERE";
const string hpcPassword = "PASSWORD_GOES_HERE";
const int slowTaskSleepInSeconds = 20;
const int taskSleepInSeconds = 3;
const int numberOfTasks = 4;
const int maxTimeToWait = (taskSleepInSeconds * 2) + 3;
const int coresPerTask = 2;
var scheduler = new Scheduler();
scheduler.Connect(hpcServer);
var jobCompletedResetEvent = new ManualResetEvent(false);
ISchedulerJob job = ((IScheduler) scheduler).CreateJob();
job.OnJobState += (sender, arg) =>
{
Console.WriteLine("jobid: " + arg.JobId + ", oldstate: " + arg.PreviousState + ", newstate: " + arg.NewState);
if (arg.NewState == JobState.Finished || arg.NewState == JobState.Failed)
{
jobCompletedResetEvent.Set();
}
};
SubmitJob(scheduler.CreateJob(), scheduler, 1, 1, slowTaskSleepInSeconds, hpcUsername, hpcPassword); // slow task that requires 1 core
var stopwatch = Stopwatch.StartNew();
var jobId = SubmitJob(job, scheduler, numberOfTasks, coresPerTask, taskSleepInSeconds, hpcUsername, hpcPassword);
Assert.IsTrue(jobCompletedResetEvent.WaitOne(TimeSpan.FromSeconds(maxTimeToWait)), "Should have completed " + numberOfTasks + "*" + taskSleepInSeconds + "s tasks requiring " + coresPerTask + " coresPerTask in " + maxTimeToWait + "s");
stopwatch.Stop();
Console.WriteLine("job completed in " + stopwatch.ElapsedMilliseconds + "ms");
var loadedJob = scheduler.OpenJob(jobId);
Console.WriteLine("jobid: " + jobId + " ran on nodes: " + String.Join(",", loadedJob.AllocatedNodes));
foreach (var result in loadedJob.GetTaskList(null, null, false).OfType<ISchedulerTask>())
{
var allocatedNodes = String.Join(",", result.AllocatedNodes);
Console.WriteLine("taskID " + result.TaskId.JobTaskId + " got result: " + result.Output.TrimEnd() + " on allocatednodes: " + allocatedNodes);
Assert.AreEqual(1, result.AllocatedNodes.Count, " Should have only run the task on 1 core but ran on "+ result.AllocatedNodes.Count + " cores (" + allocatedNodes + ")");
}
}
private static int SubmitJob(ISchedulerJob job, Scheduler scheduler, int numberOfTasks, int coresPerTask, int taskSleepInSeconds, string hpcUsername, string hpcPassword)
{
for (int i = 0; i < numberOfTasks; i++)
{
ISchedulerTask task = job.CreateTask();
task.MinimumNumberOfCores = coresPerTask;
task.MaximumNumberOfCores = coresPerTask;
task.CommandLine = "powershell start-sleep " + taskSleepInSeconds + "";
job.AddTask(task);
}
scheduler.SubmitJob(job, hpcUsername, hpcPassword);
return job.Id;
}
}
