После удаления элемента RecyclerView остается пустое место

Пишу калькулятор времени в стиле гугл калькулятора. Список слагаемых (RecyclerView) находится над кнопками, объединенные в NestedScrollView. При этом, когда свайпом удаляю один из элементов, на его месте остается пустое место. Если далее зажать другую строку или удалить ее, то это пустое место исчезнет. Не могу понять, почему этого не происходит сразу, как задумано.

MainActivity

@RequiresApi(Build.VERSION_CODES.O)
class MainActivity : AppCompatActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
    super.onCreate(savedInstanceState)
    setContentView(R.layout.activity_main)

    val imm: InputMethodManager = getSystemService(INPUT_METHOD_SERVICE) as InputMethodManager
    imm.hideSoftInputFromWindow(
        findViewById<Button>(R.id.button_plus).windowToken,
        InputMethodManager.HIDE_NOT_ALWAYS
    )

    val recyclerView = findViewById<RecyclerView>(R.id.recycler_view)
    recyclerView.setHasFixedSize(true)
    recyclerView.layoutManager = LinearLayoutManager(this)

    val adapter = RecyclerListAdapter()
    recyclerView.adapter = adapter

    val callback: ItemTouchHelper.Callback = SimpleItemTouchHelperCallback(adapter)
    val touchHelper = ItemTouchHelper(callback)
    touchHelper.attachToRecyclerView(recyclerView)

    val mLayoutManager = LinearLayoutManager(this);
    mLayoutManager.setReverseLayout(true);
    mLayoutManager.setStackFromEnd(true);
}

override fun onPostCreate(savedInstanceState: Bundle?) {
    super.onPostCreate(savedInstanceState)
    val scrollview = findViewById<NestedScrollView>(R.id.scrollView)
    scrollview.post { scrollview.fullScroll(ScrollView.FOCUS_DOWN) }
}

}

RecyclerListAdapter

class RecyclerListAdapter : RecyclerView.Adapter<ItemViewHolder>(), ItemTouchHelperAdapter {
private val mItems: MutableList<String?> = ArrayList()
override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ItemViewHolder {
    val view = LayoutInflater.from(parent.context).inflate(R.layout.list_item, parent, false)


    val mLayoutManager = LinearLayoutManager(view.context);
    mLayoutManager.setReverseLayout(true);
    mLayoutManager.setStackFromEnd(true);

    return ItemViewHolder(view)
}

override fun onBindViewHolder(holder: ItemViewHolder, position: Int) {
    holder.textView!!.text = mItems[position]
}

override fun getItemCount(): Int {
    return mItems.size
}

override fun onItemMove(fromPosition: Int, toPosition: Int): Boolean {
    if (fromPosition < toPosition) {
        for (i in fromPosition until toPosition) {
            Collections.swap(mItems, i, i + 1)
        }
    } else {
        for (i in fromPosition downTo toPosition + 1) {
            Collections.swap(mItems, i, i - 1)
        }
    }
    notifyItemMoved(fromPosition, toPosition)
    return true
}

override fun onItemDismiss(position: Int) {
    mItems.removeAt(position)
    notifyItemRemoved(position)
    notifyItemRangeChanged(position, mItems.size)
}

companion object {
    private val STRINGS = arrayOf(
        "One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten"
    )
}

init {
    mItems.addAll(Arrays.asList(*STRINGS))
}

}

ItemViewHolder

class ItemViewHolder(itemView: View?) : RecyclerView.ViewHolder(itemView!!), ItemTouchHelperViewHolder {
public val textView: TextView?

init {
    textView = itemView as TextView?
}

@SuppressLint("ResourceAsColor")
override fun onItemSelected() {
    itemView.setBackgroundColor(R.color.purple_500)
}

override fun onItemClear() {
    itemView.setBackgroundColor(0)
}

}

SimpleItemTouchHelperCallback

internal class SimpleItemTouchHelperCallback(private val mAdapter: ItemTouchHelperAdapter) :
ItemTouchHelper.Callback() {

override fun getMovementFlags(recyclerView: RecyclerView,
                              viewHolder: RecyclerView.ViewHolder): Int {

    val dragFlags = ItemTouchHelper.UP or ItemTouchHelper.DOWN
    val swipeFlags = ItemTouchHelper.START or ItemTouchHelper.END
    return makeMovementFlags(dragFlags, swipeFlags)
}

override fun onMove(recyclerView: RecyclerView, viewHolder: RecyclerView.ViewHolder, target: RecyclerView.ViewHolder): Boolean {
    mAdapter.onItemMove(
        viewHolder.adapterPosition,
        target.adapterPosition
    )
    return true
}

override fun onSwiped(viewHolder: RecyclerView.ViewHolder, direction: Int) {
    mAdapter.onItemDismiss(viewHolder.adapterPosition)

}

override fun onSelectedChanged(viewHolder: RecyclerView.ViewHolder?, actionState: Int) {
    // We only want the active item
    if (actionState != ItemTouchHelper.ACTION_STATE_IDLE) {
        if (viewHolder is ItemTouchHelperViewHolder) {
            val itemViewHolder = viewHolder as ItemTouchHelperViewHolder
            itemViewHolder.onItemSelected()
        }
    }
    super.onSelectedChanged(viewHolder, actionState)
}

override fun clearView(recyclerView: RecyclerView, viewHolder: RecyclerView.ViewHolder) {
    super.clearView(recyclerView!!, viewHolder!!)
    if (viewHolder is ItemTouchHelperViewHolder) {
        val itemViewHolder = viewHolder as ItemTouchHelperViewHolder
        itemViewHolder.onItemClear()
    }
}

override fun isLongPressDragEnabled(): Boolean {
    return true
}

override fun isItemViewSwipeEnabled(): Boolean {
    return true
}

}