Toolbar back button in Nested Navigation Graph

Question

I am not sure if this is a duplicate or not but I could not find an answer to this.

This is my app structure and I am not too sure how to get proper toolbar support with it.

Activity A (NavHostFragment) -> Fragment A -> (Nested Navigation Graph) Fragment B -> Fragment BA

And the issue is when I go into by nested navigation graph and go to a fragment from inside there the toolbar no longer shows the Up button.

My Navigation Graph

<navigation xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:id="@+id/main_navigation"
    app:startDestination="@id/loginFragment">

    <fragment
        android:id="@+id/loginFragment"
        android:name="LoginFragment"
        android:label="Login"
        tools:layout="@layout/fragment_login">
        <action
            android:id="@+id/action_loginFragment_to_overviewFragment"
            app:destination="@id/overview"
            app:exitAnim="@anim/fragment_fade_exit"
            app:popUpTo="@id/loginFragment"
            app:popUpToInclusive="true" />
    </fragment>


    <fragment
        android:id="@+id/overview"
        android:name="OverviewFragment"
        android:label="Overview"
        tools:layout="@layout/fragment_overview" />


    <fragment
        android:id="@+id/accounts"
        android:name="AccountsFragment"
        android:label="Accounts"
        tools:layout="@layout/fragment_accounts">

        <action
            android:id="@+id/openAccountDetails"
            app:destination="@id/placeholder"
            app:enterAnim="@anim/slide_in_right"
            app:exitAnim="@anim/shrink"
            app:popEnterAnim="@anim/grow"
            app:popExitAnim="@anim/slide_out_right" />

    </fragment>

    <fragment
        android:id="@+id/budget"
        android:name="BudgetsFragment"
        android:label="Budgets"
        tools:layout="@layout/fragment_budgets" />

    <fragment
        android:id="@+id/insights"
        android:name="InsightsFragment"
        android:label="Insights"
        tools:layout="@layout/fragment_insights" />


    <!--Nested Navigation For Settings Scope-->
    <navigation
        android:id="@+id/settingsNavigation"
        app:startDestination="@id/settingsFragment">
        <fragment
            android:id="@+id/settingsFragment"
            android:name="SettingsFragment"
            android:label="Settings"
            tools:layout="@layout/fragment_settings">

            <action
                android:id="@+id/openSettingsBottomSheet"
                app:destination="@+id/settingsBottomSheet" />

            <action
                android:id="@+id/openProfileFragment"
                app:destination="@id/profileFragment"
                app:enterAnim="@anim/slide_in_right"
                app:exitAnim="@anim/shrink"
                app:popEnterAnim="@anim/grow"
                app:popExitAnim="@anim/slide_out_right" />
        </fragment>
        <dialog
            android:id="@+id/settingsBottomSheet"
            android:name="SettingsThemeBottomSheet" />

        <fragment
            android:id="@+id/profileFragment"
            android:name="ProfileFragment"
            android:label="Profile"
            tools:layout="@layout/fragment_profile" />

    </navigation>


    <fragment
        android:id="@+id/placeholder"
        android:name="PlaceholderFragment"
        android:label="Placeholder"
        tools:layout="@layout/fragment_placeholder" />

</navigation>

If I pull the ProfileFragment outside into the main navigation then it works and I can get the toolbar to work but that is something I want to avoid and not do