How to use MPI_Type_create_subarray?

Question

It is clear that its arguments are:

int MPI_Type_create_subarray(
  int ndims,
  int array_of_sizes[],
  int array_of_subsizes[],
  int array_of_starts[],
  int order,
  MPI_Datatype oldtype,
  MPI_Datatype *newtype
);

However, I cannot understand how this method receives the original array which we want to split and where it returns the new subarray (as this method should return an integer). In other words, I simply would like to see a simple implementation of this method in C++, which I am not able to find on the Internet.

Answer 1

MPI_Type_create_subarray() neither takes an original array nor returns a subarray; it creates an MPI type which describes the memory layout of a subarray given: a larger array of some given type; a set of subsizes; and a "corner" from which to start.

You can then use this newly created MPI type to extract just the data you want from any appropriately-sized array and send it in a message to another task (with point-to-point message passing routines), all other tasks (via collectives), or write it to disk (with MPI-IO). In the following example, rank 0 uses an MPI subarray type to extract a subarray from a larger array of integers and sends it to rank 1. Rank 1, just receiving it into a contiguous buffer, doesn't need to receive it as any special type; it just receives the data as so many integers.

#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>

void printarr(int **data, int n, char *str);
int **allocarray(int n);

int main(int argc, char **argv) {

    /* array sizes */
    const int bigsize =10;
    const int subsize =5;

    /* communications parameters */
    const int sender  =0;
    const int receiver=1;
    const int ourtag  =2;

    int rank, size;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    if (size < receiver+1) {
        if (rank == 0)
            fprintf(stderr,"%s: Needs at least %d  processors.\n", argv[0], receiver+1);
        MPI_Finalize();
        return 1;
    }

    if (rank == sender) {
        int **bigarray = allocarray(bigsize);
        for (int i=0; i<bigsize; i++)
            for (int j=0; j<bigsize; j++)
                bigarray[i][j] = i*bigsize+j;


        printarr(bigarray, bigsize, " Sender: Big array ");

        MPI_Datatype mysubarray;

        int starts[2] = {5,3};
        int subsizes[2]  = {subsize,subsize};
        int bigsizes[2]  = {bigsize, bigsize};
        MPI_Type_create_subarray(2, bigsizes, subsizes, starts,
                                 MPI_ORDER_C, MPI_INT, &mysubarray);
        MPI_Type_commit(&mysubarray);

        MPI_Send(&(bigarray[0][0]), 1, mysubarray, receiver, ourtag, MPI_COMM_WORLD);
        MPI_Type_free(&mysubarray);

        free(bigarray[0]);
        free(bigarray);

    } else if (rank == receiver) {

        int **subarray = allocarray(subsize);

        for (int i=0; i<subsize; i++)
            for (int j=0; j<subsize; j++)
                subarray[i][j] = 0;

        MPI_Recv(&(subarray[0][0]), subsize*subsize, MPI_INT, sender, ourtag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);

        printarr(subarray, subsize, " Receiver: Subarray -- after receive");

        free(subarray[0]);
        free(subarray);
    }

    MPI_Finalize();
    return 0;
}

void printarr(int **data, int n, char *str) {    
    printf("-- %s --\n", str);
    for (int i=0; i<n; i++) {
        for (int j=0; j<n; j++) {
            printf("%3d ", data[i][j]);
        }
        printf("\n");
    }
}

int **allocarray(int n) {
    int *data = malloc(n*n*sizeof(int));
    int **arr = malloc(n*sizeof(int *));
    for (int i=0; i<n; i++)
        arr[i] = &(data[i*n]);

    return arr;
}

Running this gives

$ mpicc -o subarray subarray.c  -std=c99 -Wall -g
$ mpirun -np 2 ./subarray
    --  Sender: Big array  --
  0   1   2   3   4   5   6   7   8   9 
 10  11  12  13  14  15  16  17  18  19 
 20  21  22  23  24  25  26  27  28  29 
 30  31  32  33  34  35  36  37  38  39 
 40  41  42  43  44  45  46  47  48  49 
 50  51  52  53  54  55  56  57  58  59 
 60  61  62  63  64  65  66  67  68  69 
 70  71  72  73  74  75  76  77  78  79 
 80  81  82  83  84  85  86  87  88  89 
 90  91  92  93  94  95  96  97  98  99 
--  Receiver: Subarray -- after receive --
 53  54  55  56  57 
 63  64  65  66  67 
 73  74  75  76  77 
 83  84  85  86  87 
 93  94  95  96  97