I know it is a little late, but I think I have found a way to resize without "wriggle" (inside window drawing lag will still remain).
Unlike what manuell has said, WM_NCCALCSIZE is the root of all evil. Also this method should work with any window style (tested with WS_POPUP and WS_OVERLAPPEDWINDOW)
while preserving their functionality, so it's time for me to shut up and post the code with commentary:
//some sizing border definitions
#define MINX 200
#define MINY 200
#define BORDERWIDTH  5
#define TITLEBARWIDTH  30
//................
HWND TempHwnd = Create(NULL, TEXT("CUSTOM BORDER"), TEXT("CUSTOM BORDER"),
               WS_POPUP | WS_VISIBLE,
               100, 100, 400, 400, NULL, NULL, 
                       GetModuleHandle(NULL), NULL);
//...............
LRESULT CALLBACK WinMsgHandler(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
    {
        switch (uMsg)
        {
        case WM_SIZING: // I use this message to redraw window on sizing (o rly?)
            RedrawWindow(hWnd, NULL, NULL, RDW_INVALIDATE | RDW_NOERASE | RDW_INTERNALPAINT);
            return DefWindowProc(hWnd, uMsg, wParam, lParam);
        case WM_PAINT: // Used to draw borders and stuff to test WM_NCHITTEST
            {
                PAINTSTRUCT ps;
                BeginPaint(hWnd, &ps);
                RECT ClientRect;
                GetClientRect(hWnd, &ClientRect);
                RECT BorderRect = { BORDERWIDTH, BORDERWIDTH, ClientRect.right - BORDERWIDTH - BORDERWIDTH, ClientRect.bottom - BORDERWIDTH - BORDERWIDTH },
                     TitleRect = { BORDERWIDTH, BORDERWIDTH, ClientRect.right - BORDERWIDTH - BORDERWIDTH, TITLEBARWIDTH };
                HBRUSH BorderBrush = CreateSolidBrush(0x0000ff);
                FillRect(ps.hdc, &ClientRect, BorderBrush);
                FillRect(ps.hdc, &BorderRect, GetSysColorBrush(2));
                FillRect(ps.hdc, &TitleRect, GetSysColorBrush(1));
                DeleteObject(BorderBrush);
                EndPaint(hWnd, &ps);
            }
            break;
        case WM_GETMINMAXINFO: // It is used to restrict WS_POPUP window size
            {              // I don't know if this works on others
                MINMAXINFO *min_max = reinterpret_cast<MINMAXINFO *>(lParam);
                min_max->ptMinTrackSize.x = MINX;
                min_max->ptMinTrackSize.y = MINY;
            }
            break;
        case WM_CREATE:  // In this message we use MoveWindow to invoke
            {        //WM_NCCALCSIZE msg to remove border
                CREATESTRUCT *WindowInfo = reinterpret_cast<CREATESTRUCT *>(lParam);
                MoveWindow(hWnd, WindowInfo->x, WindowInfo->y, WindowInfo->cx - BORDERWIDTH, WindowInfo->cy - BORDERWIDTH, TRUE); 
//Notice that "- BORDERWIDTH" is recommended on every manually called resize function,
//Because we will add BORDERWIDTH value in WM_NCCALCSIZE message
            }
            break;
        case WM_NCCALCSIZE:
            { // Microsoft mentioned that if wParam is true, returning 0 should be enough, but after MoveWindow or similar functions it would begin to "wriggle"
                if (wParam)
                {
                    NCCALCSIZE_PARAMS *Params = reinterpret_cast<NCCALCSIZE_PARAMS *>(lParam);
                    Params->rgrc[0].bottom += BORDERWIDTH; // rgrc[0] is what makes this work, don't know what others (rgrc[1], rgrc[2]) do, but why not change them all?
                    Params->rgrc[0].right += BORDERWIDTH;
                    Params->rgrc[1].bottom += BORDERWIDTH;
                    Params->rgrc[1].right += BORDERWIDTH;
                    Params->rgrc[2].bottom += BORDERWIDTH;
                    Params->rgrc[2].right += BORDERWIDTH;
                    return 0;
                }
                return DefWindowProc(hWnd, uMsg, wParam, lParam);
            }
            break;
        case WM_NCHITTEST:
            {
                RECT WindowRect;
                int x, y;
                GetWindowRect(hWnd, &WindowRect);
                x = GET_X_LPARAM(lParam) - WindowRect.left;
                y = GET_Y_LPARAM(lParam) - WindowRect.top;
                if (x >= BORDERWIDTH && x <= WindowRect.right - WindowRect.left - BORDERWIDTH && y >= BORDERWIDTH && y <= TITLEBARWIDTH)
                    return HTCAPTION;
                else if (x < BORDERWIDTH && y < BORDERWIDTH)
                    return HTTOPLEFT;
                else if (x > WindowRect.right - WindowRect.left - BORDERWIDTH && y < BORDERWIDTH)
                    return HTTOPRIGHT;
                else if (x > WindowRect.right - WindowRect.left - BORDERWIDTH && y > WindowRect.bottom - WindowRect.top - BORDERWIDTH)
                    return HTBOTTOMRIGHT;
                else if (x < BORDERWIDTH && y > WindowRect.bottom - WindowRect.top - BORDERWIDTH)
                    return HTBOTTOMLEFT;
                else if (x < BORDERWIDTH)
                    return HTLEFT;
                else if (y < BORDERWIDTH)
                    return HTTOP;
                else if (x > WindowRect.right - WindowRect.left - BORDERWIDTH)
                    return HTRIGHT;
                else if (y > WindowRect.bottom - WindowRect.top - BORDERWIDTH)
                    return HTBOTTOM;
                else
                    return HTCLIENT;
            }
            break;
        default:
            return DefWindowProc(hWnd, uMsg, wParam, lParam);
        }
        return 0;
    }