So, in my C# app I have the following code to encrypt the data:
using System;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Xml.Serialization;
namespace TEST
{
    public class RSAEncrypter
    {
        private static RSACryptoServiceProvider RSA;
        private RSAParameters _privateKey;
        private RSAParameters _publicKey;
        public RSAEncrypter()
        {
            RSA = new RSACryptoServiceProvider(2048);
            _privateKey = RSA.ExportParameters(true);
            _publicKey = RSA.ExportParameters(false);
        }
        public string Encrypt(string plainText, string publicKey)
        {
            RSA.FromXmlString(publicKey);
            _publicKey = RSA.ExportParameters(false);
            RSA.ImportParameters(_publicKey);
            var data = Encoding.Unicode.GetBytes(plainText);
            var cypher = RSA.Encrypt(data, false);
            return Convert.ToBase64String(cypher);
        }
        // Gets the privatekey as a string
        public string PrivateKeyString()
        {
            var sw = new StringWriter();
            var xs = new XmlSerializer(typeof(RSAParameters));
            xs.Serialize(sw, _privateKey);
            return sw.ToString();
        }
    }
}
Which gives me, for example, a key in the following format (as a string):
<?xml version="1.0" encoding="utf-16"?>
<RSAParameters xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
  <Exponent>AQAB</Exponent>
  <Modulus>38Z4+7H1ADzMPO8z5+QdxXS21YBEaq9Xacf7dHFXUpK72SUAIYnfijc5RDSgGFismTNlrrOa7m/6+iIWS/yB7+esvIjgfSFm+QU2aeC16NisMuw+KvPeEr8CVMjh8F5YW1ST4qKXHXG6qIe/FM2LPVGV92O9WO1ATIDcATO8UU2rJgrxKMdmE9fawqmy/j7fwI1+FL6LCNgdvgZ3OOLLwHVcyOyj7ibiIUQAcw10qW0I4MBnQL5V8udKrhKXKoVE6rsfLZoBC9rBD62ckB7CJfMsGcAVffBvnd7SRJiTFEEPVZFqzyGk0BOeqbJkHbzKNytNkUjnFQlDX9tSLCtufQ==</Modulus>
  <P>+jlZUY/lv0bMWbcWVXZ+moDF96ZQ+uKHMbpAMN18ByRmZSLZ9CeHTNEkoudOzB6R9wN8xyEmnCrDZujSB65uBILdJvsJk8TYBThp5RBbBZ1YQb30CcxhsMX2s6Gwze8CoXmBU5as6tMDh+tBpxxeYxE/crS3rP7kAM2Lr88cILM=</P>
  <Q>5PDUdn/RB8dhxkjSFZS0MtrefVgYmoDNjN9O+Ru7ZzNz7eqig1zLlKytDQzmpaIv6Zvrn5Y5TBaFDp1BTVLAStu8e/RU4i7qWVu6Xm83xtgB4NDULXStyYqpeZhyilD4BsfIYeRyZ3A9wUyACQ6CU3GxuIczWYkfT3HmSy5ebA8=</Q>
  <DP>nEH/8x0nXeF6b3QUMF6FBTrxZYuo+mNIBdfHijxl3ZfvkazH6t5cca4RcOF9pZ5ZjKXS4A9lqxRRXgx6TG2zKoIGVPdjrbG5LNlj17X1AXaWzMcwhIXrY5bcTqTkYlWlkOztxCNN7H7Fr7VMFG10y+zTcHBGW3P5Mj8pwipV6F0=</DP>
  <DQ>DMkWVHfW6KRN5ZDzipj/Z0ep3T4qQZan5BIkiuztjlnlQ4gzAzsPc4IhN/VcfCuOmXFHu2XcVU98ptBJcVQJwSR8Zj/C7c7I76ybv+JeLxCpKjD/aHp3qiXASTYmT2suLtLBchYb/YLbMAxhqh/RT2+uCSwjxgBOa1VlExXH2Ck=</DQ>
  <InverseQ>zlFNPMXsmgjWz0rA5y3stzoKDiw0mbb1rU6iSXKKOZz5djokAX1tjtcwUbzdv3I5x8m+3K1qprXURUYbZXf+ubwkoYiG4WaJDWhZ+y33Q8iK87BUKy6Q062F4XryBVagCJHKMNStiiMmFB9IdbDu/wbYh52BjrwhwncLOEYaSaQ=</InverseQ>
  <D>YQvsEAv/WtkDIjIC6sBtgOK7ICB+i137pO6LyNYWrsLgIK4BPopSndiRR1kjTSu3vsEhigBuYpXB3JTH4rBhka+BpEogQWQpCjoOfSBtA8xj8bmuxGX6m1qnIin0gpAH9aPaduFYc/aMouYsIlN53V/yj9V7moNZ7VO9FfBf7UnX7yTc2/fvXz9atXepQmir2YOO41Z5KgytSF1M7R3eZq1GLTnaqRK+LCQ5cKkxqcIAtUX7BZgkLnHZs3zqLo998gXbogmkqxB4NWc9bwWjybC1CMWyZeiRuaw/Wye4GbNmpM7sfsB7GINndwCSqfJzwSetGCSca8x2wkms+5zOaQ==</D>
</RSAParameters>
So how could I decrypt a message in PHP using this key?
I found this in PHP: openssl_private_decrypt but it doesn't accept the key in this format apparently.