Overwriting measureChildWithMargins is the way to go, however to get it actually working you need custom LayoutParams so you can "carry over" data from your adapter or straight up inflate your desired percentages out of XML.
This layout manager will handle it:
open class PercentLinearLayoutManager : LinearLayoutManager {
    constructor(context: Context?) : super(context)
    constructor(context: Context?, orientation: Int, reverseLayout: Boolean) : super(context, orientation, reverseLayout)
    constructor(context: Context?, attrs: AttributeSet?, defStyleAttr: Int, defStyleRes: Int) : super(context, attrs, defStyleAttr, defStyleRes)
    // where we actually force view to be measured based on custom layout params
    override fun measureChildWithMargins(child: View, widthUsed: Int, heightUsed: Int) {
        val pp = child.layoutParams as PercentParams
        if (pp.maxPercentHeight <= 0.0f)
            super.measureChildWithMargins(child, widthUsed, heightUsed)
        else {
            val widthSpec = getChildMeasureSpec(width, widthMode,
                    paddingLeft + paddingRight + pp.leftMargin + pp.rightMargin + widthUsed, pp.width,
                    canScrollHorizontally())
            val maxHeight = (height * pp.maxPercentHeight).toInt()
            val heightSpec = when (pp.height) {
                ViewGroup.LayoutParams.MATCH_PARENT -> View.MeasureSpec.makeMeasureSpec(maxHeight, View.MeasureSpec.EXACTLY)
                ViewGroup.LayoutParams.WRAP_CONTENT -> View.MeasureSpec.makeMeasureSpec(maxHeight, View.MeasureSpec.AT_MOST)
                else -> View.MeasureSpec.makeMeasureSpec(min(pp.height, maxHeight), View.MeasureSpec.AT_MOST)
            }
            child.measure(widthSpec, heightSpec)
        }
    }
    // everything below is needed to generate custom params
    override fun checkLayoutParams(lp: RecyclerView.LayoutParams) = lp is PercentParams
    override fun generateDefaultLayoutParams(): RecyclerView.LayoutParams {
        return PercentParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT)
    }
    override fun generateLayoutParams(lp: ViewGroup.LayoutParams): RecyclerView.LayoutParams {
        return PercentParams(lp)
    }
    override fun generateLayoutParams(c: Context, attrs: AttributeSet): RecyclerView.LayoutParams {
        return PercentParams(c, attrs)
    }
    class PercentParams : RecyclerView.LayoutParams {
        /** Max percent height of recyclerview this item can have. If height is `match_parent` this size is enforced. */
        var maxPercentHeight = 0.0f
        constructor(c: Context, attrs: AttributeSet) : super(c, attrs){
            val t = c.obtainStyledAttributes(attrs, R.styleable.PercentLinearLayoutManager_Layout)
            maxPercentHeight = t.getFloat(R.styleable.PercentLinearLayoutManager_Layout_maxPercentHeight, 0f)
            t.recycle()
        }
        constructor(width: Int, height: Int) : super(width, height)
        constructor(source: MarginLayoutParams?) : super(source)
        constructor(source: ViewGroup.LayoutParams?) : super(source)
        constructor(source: RecyclerView.LayoutParams?) : super(source)
    }
}
To handle inflation from XML custom attribute is needed, add it to values/attrs.xml:
<declare-styleable name="PercentLinearLayoutManager_Layout">
    <attr name="maxPercentHeight" format="float"/>
</declare-styleable>
Then you have two options:
1 - alter custom params inside onBindViewHolder to have per-item control:
val lp = holder.itemView.layoutParams as PercentLinearLayoutManager.PercentParams
if(modifyThisViewHolderHeight) {
    lp.maxPercentHeight = 0.6f
} else {
    lp.maxPercentHeight = 0.0f // clear percentage in case viewholder is reused
}
2 - use custom attribute inside your viewholder layout:
<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="wrap_content"
    android:layout_height="wrap_content"
    app:maxPercentHeight ="0.6">
    <!-- rest of layout-->
</FrameLayout>