Starting in .NET Core 3.0, this is (largely) built-in.
Writing SubjectPublicKeyInfo and RSAPrivateKey
.NET Core 3.0 built-in API
The output of the builtin API is the binary representation, to make them PEM you need to output the header, footer, and base64:
private static string MakePem(byte[] ber, string header)
{
    StringBuilder builder = new StringBuilder("-----BEGIN ");
    builder.Append(header);
    builder.AppendLine("-----");
    string base64 = Convert.ToBase64String(ber);
    int offset = 0;
    const int LineLength = 64;
    while (offset < base64.Length)
    {
        int lineEnd = Math.Min(offset + LineLength, base64.Length);
        builder.AppendLine(base64.Substring(offset, lineEnd - offset));
        offset = lineEnd;
    }
    builder.Append("-----END ");
    builder.Append(header);
    builder.AppendLine("-----");
    return builder.ToString();
}
So to produce the strings:
string publicKey = MakePem(rsa.ExportSubjectPublicKeyInfo(), "PUBLIC KEY");
string privateKey = MakePem(rsa.ExportRSAPrivateKey(), "RSA PRIVATE KEY");
Semi-manually
If you can't use .NET Core 3.0, but you can use pre-release NuGet packages, you can make use of the prototype ASN.1 writer package (which is the same code that's used internally in .NET Core 3.0; it's just that the API surface isn't finalized).
To make the public key:
private static string ToSubjectPublicKeyInfo(RSA rsa)
{
    RSAParameters rsaParameters = rsa.ExportParameters(false);
    AsnWriter writer = new AsnWriter(AsnEncodingRules.DER);
    writer.PushSequence();
    writer.PushSequence();
    writer.WriteObjectIdentifier("1.2.840.113549.1.1.1");
    writer.WriteNull();
    writer.PopSequence();
    AsnWriter innerWriter = new AsnWriter(AsnEncodingRules.DER);
    innerWriter.PushSequence();
    WriteRSAParameter(innerWriter, rsaParameters.Modulus);
    WriteRSAParameter(innerWriter, rsaParameters.Exponent);
    innerWriter.PopSequence();
    writer.WriteBitString(innerWriter.Encode());
    writer.PopSequence();
    return MakePem(writer.Encode(), "PUBLIC KEY");
}
And to make the private key:
private static string ToRSAPrivateKey(RSA rsa)
{
    RSAParameters rsaParameters = rsa.ExportParameters(true);
    AsnWriter writer = new AsnWriter(AsnEncodingRules.DER);
    writer.PushSequence();
    writer.WriteInteger(0);
    WriteRSAParameter(writer, rsaParameters.Modulus);
    WriteRSAParameter(writer, rsaParameters.Exponent);
    WriteRSAParameter(writer, rsaParameters.D);
    WriteRSAParameter(writer, rsaParameters.P);
    WriteRSAParameter(writer, rsaParameters.Q);
    WriteRSAParameter(writer, rsaParameters.DP);
    WriteRSAParameter(writer, rsaParameters.DQ);
    WriteRSAParameter(writer, rsaParameters.InverseQ);
    writer.PopSequence();
    return MakePem(writer.Encode(), "RSA PRIVATE KEY");
}
Reading them back
.NET Core 3.0 built-in API
Except that .NET Core 3.0 doesn't understand PEM encoding, so you have to do PEM->binary yourself:
private const string RsaPrivateKey = "RSA PRIVATE KEY";
private const string SubjectPublicKeyInfo = "PUBLIC KEY";
private static byte[] PemToBer(string pem, string header)
{
    // Technically these should include a newline at the end,
    // and either newline-or-beginning-of-data at the beginning.
    string begin = $"-----BEGIN {header}-----";
    string end = $"-----END {header}-----";
    int beginIdx = pem.IndexOf(begin);
    int base64Start = beginIdx + begin.Length;
    int endIdx = pem.IndexOf(end, base64Start);
    return Convert.FromBase64String(pem.Substring(base64Start, endIdx - base64Start));
}
Once that's done you can now load the keys:
using (RSA rsa = RSA.Create())
{
    rsa.ImportRSAPrivateKey(PemToBer(pemPrivateKey, RsaPrivateKey), out _);
    ...
}
using (RSA rsa = RSA.Create())
{
    rsa.ImportSubjectPublicKeyInfo(PemToBer(pemPublicKey, SubjectPublicKeyInfo), out _);
    ...
}
Semi-manually
If you can't use .NET Core 3.0, but you can use pre-release NuGet packages, you can make use of the prototype ASN.1 reader package (which is the same code that's used internally in .NET Core 3.0; it's just that the API surface isn't finalized).
For the public key:
private static RSA FromSubjectPublicKeyInfo(string pem)
{
    AsnReader reader = new AsnReader(PemToBer(pem, SubjectPublicKeyInfo), AsnEncodingRules.DER);
    AsnReader spki = reader.ReadSequence();
    reader.ThrowIfNotEmpty();
    AsnReader algorithmId = spki.ReadSequence();
    if (algorithmId.ReadObjectIdentifierAsString() != "1.2.840.113549.1.1.1")
    {
        throw new InvalidOperationException();
    }
    algorithmId.ReadNull();
    algorithmId.ThrowIfNotEmpty();
    AsnReader rsaPublicKey = spki.ReadSequence();
    RSAParameters rsaParameters = new RSAParameters
    {
        Modulus = ReadNormalizedInteger(rsaPublicKey),
        Exponent = ReadNormalizedInteger(rsaPublicKey),
    };
    rsaPublicKey.ThrowIfNotEmpty();
    RSA rsa = RSA.Create();
    rsa.ImportParameters(rsaParameters);
    return rsa;
}
private static byte[] ReadNormalizedInteger(AsnReader reader)
{
    ReadOnlyMemory<byte> memory = reader.ReadIntegerBytes();
    ReadOnlySpan<byte> span = memory.Span;
    if (span[0] == 0)
    {
        span = span.Slice(1);
    }
    return span.ToArray();
}
And because the private key values have to have the correct size arrays, the private key one is just a little trickier:
private static RSA FromRSAPrivateKey(string pem)
{
    AsnReader reader = new AsnReader(PemToBer(pem, RsaPrivateKey), AsnEncodingRules.DER);
    AsnReader rsaPrivateKey = reader.ReadSequence();
    reader.ThrowIfNotEmpty();
    if (!rsaPrivateKey.TryReadInt32(out int version) || version != 0)
    {
        throw new InvalidOperationException();
    }
    byte[] modulus = ReadNormalizedInteger(rsaPrivateKey);
    int halfModulusLen = (modulus.Length + 1) / 2;
    RSAParameters rsaParameters = new RSAParameters
    {
        Modulus = modulus,
        Exponent = ReadNormalizedInteger(rsaPrivateKey),
        D = ReadNormalizedInteger(rsaPrivateKey, modulus.Length),
        P = ReadNormalizedInteger(rsaPrivateKey, halfModulusLen),
        Q = ReadNormalizedInteger(rsaPrivateKey, halfModulusLen),
        DP = ReadNormalizedInteger(rsaPrivateKey, halfModulusLen),
        DQ = ReadNormalizedInteger(rsaPrivateKey, halfModulusLen),
        InverseQ = ReadNormalizedInteger(rsaPrivateKey, halfModulusLen),
    };
    rsaPrivateKey.ThrowIfNotEmpty();
    RSA rsa = RSA.Create();
    rsa.ImportParameters(rsaParameters);
    return rsa;
}
private static byte[] ReadNormalizedInteger(AsnReader reader, int length)
{
    ReadOnlyMemory<byte> memory = reader.ReadIntegerBytes();
    ReadOnlySpan<byte> span = memory.Span;
    if (span[0] == 0)
    {
        span = span.Slice(1);
    }
    byte[] buf = new byte[length];
    int skipSize = length - span.Length;
    span.CopyTo(buf.AsSpan(skipSize));
    return buf;
}