Why doesn't this work?  I know it's not using EF4, but the concept is the same.
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Text;
    using System.Threading.Tasks;
    namespace LinqOrderByReference
    {
        class Program
        {
            static void Main(string[] args)
            {
                EntityA[] entitiesA = {   new EntityA { Ordinal = 3, SecretCode = "xxx"}, 
                                          new EntityA { Ordinal = 2 ,SecretCode = "x"}, 
                                          new EntityA { Ordinal = 1 ,SecretCode = "x"} };
                EntityB[] entitiesB = {  new EntityB { Name = "C", EntityA = entitiesA[2] }, 
                                          new EntityB {Name = "B", EntityA = entitiesA[1] },
                                          new EntityB {Name = "A", EntityA = entitiesA[0] } };
                IEnumerable<EntityB> entitiesBList = entitiesB.OrderBy(x => x.EntityA.Ordinal);
                foreach (EntityB b in entitiesBList)
                {
                    Console.WriteLine("EntityB.Name: {0}, Ordinal {1}.", b.Name, b.EntityA.Ordinal);
                }
                Console.Read();
            }
        }
        public class EntityA
        {
            public int Ordinal { get; set; }
            public string SecretCode { get; set; }
        }
        public class EntityB
        {
            public string Name { get; set; }
            public EntityA EntityA { get; set; }
        }
    }
Output is: 
EntityB.Name: C, Ordinal 1. 
EntityB.Name: B, Ordinal 2. 
EntityB.Name: A, Ordinal 3.