You should be able to use something like this, but you need to look at the way IE opens new windows, because sometimes it can open them in a new tab, which is probably not what you want. I use a html container to host the dialog in an iframe.
openDialog: function (dialogid, entityname, objectid, callback) {
if (objectid.length > 36) objectid = objectid.substr(1, 36);
//format the url access to dialog
var dlgUrl = String.format('{0}/cs/dialog/rundialog.aspx?DialogId=%7b{1}%7d&EntityName={2}&ObjectId=%7b{3}%7d',
Xrm.Page.context.getClientUrl(), dialogid, entityname, objectid);
// add the callback function.toString to the querystring
if (callback) dlgUrl += '&callback=' + JSON.stringify({ fnc: callback }, function (key, value) { return (typeof value === 'function') ? value.toString() : value; });
var dlg = Xrm.Utility.openWebResource('xxx_container.htm', encodeURIComponent(dlgUrl), 620, 510); //open container webresource
return dlg;
}