Copying from cuda 3D memory to linear memory: copied data is not where I expected

Question

Here is my issue:

I have a 3D array of float3 on my device:

int size[3] = {416,464,512};
cudaExtent extent = make_cudaExtent(size[0]*sizeof(float3),size[1],size[2]);
cudaPitchedPtr renderedVolume;
int ret = cudaMalloc3D(&renderedVolume, extent);
size_t pitch = renderedVolume.pitch; //pitch = 5,120
size_t slicePitch = pitch * size[1]; //slicePitch = 2,375,680

Then I work with it and make it full of outstanding data.

After that I wish to copy it on a 1D linear memory on my host:

float *host_memory = (float*)malloc(size[0]*size[1]*size[2]*sizeof(float3));
cudaMemcpy3DParms p = {0};
p.srcPtr = renderedVolume;
p.dstPtr = make_cudaPitchedPtr(host_memory,size[0]*sizeof(float3),size[0],size[1]); 
p.extent = make_cudaExtent(size[0]*sizeof(float3),size[1],size[2]);
p.srcPos = make_cudaPos(0,0,0);
p.dstPos = make_cudaPos(0,0,0);
p.kind=cudaMemcpyDeviceToHost;
cudaMemcpy3D(&p);

I am comparing the result in host_memory with the data I initially wrote tu renderedVolume (my_data) and with the data I read in my 3Dmemory, slice by slice:

float* test1 = (float*)malloc(size[0]*size[1]*sizeof(float3));
cudaMemcpy(test1, myData, size[0]*size[1]*sizeof(float3) , cudaMemcpyDeviceToHost);
float* test2 = (float*)malloc(size[0]*size[1]*sizeof(float3));
cudaMemcpy(test2,(char*)renderedVolume.ptr + slicePitch * i,size[0]*size[1]*sizeof(float3), cudaMemcpyDeviceToHost);

Problem:

• The first slice (i=0) is ok, I have the same data in host_memory, test1 and test2.
• In the second slice, I have the same data in test1 and test2. However, I should find this data in host_memory+579072 (=number of float per slice, also heigth*pitch of the destination pitched pointer) and I find it in host_memory+577504. It is off by 1568 bytes, which corresponds to nothing that I am aware of, and this is why I would very much appreciate if any of you have an idea of what the problem might be in my code ?

Answer 1

This is a late answer provided to remove this question from the unanswered list.

Below, I'm providing a full code showing how to allocate 3D memory by cudaMalloc3D, moving a host allocated 1D memory to 3D device memory by cudaMemcpy3D, performing some operations on the 3D device data by the test_kernel_3D __global__ function and moving the 3D result data back to 1D host memory, again by cudaMemcpy3D.

The __global__ function test_kernel_3D squares each element of the 3D device memory. In particular, each thread of a 2D grid takes care of performing a for loop along the "depth" dimension.

#include<stdio.h>
#include<cuda.h>
#include<cuda_runtime.h>
#include<device_launch_parameters.h>
#include<conio.h>

#define BLOCKSIZE_x 16
#define BLOCKSIZE_y 16

#define N 128
#define M 64
#define W 16

/*****************/
/* CUDA MEMCHECK */
/*****************/
#define gpuErrchk(ans) { gpuAssert((ans), __FILE__, __LINE__); }
inline void gpuAssert(cudaError_t code, char *file, int line, bool abort=true)
{
    if (code != cudaSuccess) 
    {
        fprintf(stderr,"GPUassert: %s %s %d\n", cudaGetErrorString(code), file, line);
        if (abort) { getch(); exit(code); }
    }
}

/*******************/
/* iDivUp FUNCTION */
/*******************/
int iDivUp(int a, int b){ return ((a % b) != 0) ? (a / b + 1) : (a / b); }

/******************/
/* TEST KERNEL 3D */
/******************/
__global__ void test_kernel_3D(cudaPitchedPtr devPitchedPtr)
{
    int tidx =  blockIdx.x*blockDim.x+threadIdx.x;
    int tidy =  blockIdx.y*blockDim.y+threadIdx.y;

    char* devPtr = (char*) devPitchedPtr.ptr;
    size_t pitch = devPitchedPtr.pitch;
    size_t slicePitch = pitch * N;

    for (int w = 0; w < W; w++) {
        char* slice = devPtr + w * slicePitch;
        float* row = (float*)(slice + tidy * pitch);
        row[tidx] = row[tidx] * row[tidx];
    }
}

/********/
/* MAIN */
/********/
int main()
{
    float a[N][M][W];

    for (int i=0; i<N; i++)
        for (int j=0; j<M; j++) 
            for (int w=0; w<W; w++) {
                a[i][j][w] = 3.f;
                //printf("row %i column %i depth %i value %f \n",i,j,w,a[i][j][w]);
            }

    // --- 3D pitched allocation and host->device memcopy
    cudaExtent extent = make_cudaExtent(M * sizeof(float), N, W);

    cudaPitchedPtr devPitchedPtr;
    gpuErrchk(cudaMalloc3D(&devPitchedPtr, extent));

    cudaMemcpy3DParms p = { 0 };
    p.srcPtr.ptr = a;
    p.srcPtr.pitch = M * sizeof(float);
    p.srcPtr.xsize = M;
    p.srcPtr.ysize = N;
    p.dstPtr.ptr = devPitchedPtr.ptr;
    p.dstPtr.pitch = devPitchedPtr.pitch;
    p.dstPtr.xsize = M;
    p.dstPtr.ysize = N;
    p.extent.width = M * sizeof(float);
    p.extent.height = N;
    p.extent.depth = W;
    p.kind = cudaMemcpyHostToDevice;
    gpuErrchk(cudaMemcpy3D(&p));

    dim3 GridSize(iDivUp(M,BLOCKSIZE_x),iDivUp(N,BLOCKSIZE_y));
    dim3 BlockSize(BLOCKSIZE_y,BLOCKSIZE_x);
    test_kernel_3D<<<GridSize,BlockSize>>>(devPitchedPtr);
    gpuErrchk(cudaPeekAtLastError());
    gpuErrchk(cudaDeviceSynchronize());

    p.srcPtr.ptr = devPitchedPtr.ptr;
    p.srcPtr.pitch = devPitchedPtr.pitch;
    p.dstPtr.ptr = a;
    p.dstPtr.pitch = M * sizeof(float); 
    p.kind = cudaMemcpyDeviceToHost;
    gpuErrchk(cudaMemcpy3D(&p));

    for (int i=0; i<N; i++) 
        for (int j=0; j<M; j++) 
            for (int w=0; w<W; w++)
                printf("row %i column %i depth %i value %f\n",i,j,w,a[i][j][w]);

    getch();
    return 0;
}