Take a look at the seriation package.
Starting with a block diagonal matrix:
m <- diag(1:5) %x% matrix(1, 2, 2)
m
#>       [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10]
#>  [1,]    1    1    0    0    0    0    0    0    0     0
#>  [2,]    1    1    0    0    0    0    0    0    0     0
#>  [3,]    0    0    2    2    0    0    0    0    0     0
#>  [4,]    0    0    2    2    0    0    0    0    0     0
#>  [5,]    0    0    0    0    3    3    0    0    0     0
#>  [6,]    0    0    0    0    3    3    0    0    0     0
#>  [7,]    0    0    0    0    0    0    4    4    0     0
#>  [8,]    0    0    0    0    0    0    4    4    0     0
#>  [9,]    0    0    0    0    0    0    0    0    5     5
#> [10,]    0    0    0    0    0    0    0    0    5     5
Shuffle it:
mm <- m[sample(1:10), sample(1:10)]
mm
#>       [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10]
#>  [1,]    0    0    0    0    2    0    0    0    0     2
#>  [2,]    0    0    0    0    0    5    5    0    0     0
#>  [3,]    0    3    3    0    0    0    0    0    0     0
#>  [4,]    0    0    0    0    2    0    0    0    0     2
#>  [5,]    0    0    0    4    0    0    0    0    4     0
#>  [6,]    0    0    0    4    0    0    0    0    4     0
#>  [7,]    0    0    0    0    0    5    5    0    0     0
#>  [8,]    1    0    0    0    0    0    0    1    0     0
#>  [9,]    0    3    3    0    0    0    0    0    0     0
#> [10,]    1    0    0    0    0    0    0    1    0     0
Seriate the shuffled matrix:
ser <- seriation::seriate(mm)
mm[ser[[1]], ser[[2]]]
#>       [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10]
#>  [1,]    5    5    0    0    0    0    0    0    0     0
#>  [2,]    5    5    0    0    0    0    0    0    0     0
#>  [3,]    0    0    1    1    0    0    0    0    0     0
#>  [4,]    0    0    1    1    0    0    0    0    0     0
#>  [5,]    0    0    0    0    2    2    0    0    0     0
#>  [6,]    0    0    0    0    2    2    0    0    0     0
#>  [7,]    0    0    0    0    0    0    3    3    0     0
#>  [8,]    0    0    0    0    0    0    3    3    0     0
#>  [9,]    0    0    0    0    0    0    0    0    4     4
#> [10,]    0    0    0    0    0    0    0    0    4     4