confusion on __block NSObject *obj and block runtime

Question

I use clang -rewrite-objc Block.m to generate c++ code of Block.m.

The code in Block.m is under ARC:

void func() {
    __block NSObject *obj = [[NSObject alloc] init];
    void (^blk)(void) = ^() {
        obj = nil;
    };
}

I believe when block is copied and move to the heap, the block in the heap will retain the obj. But after digging into the source code of block runtime, I got a opposite result.

The generated c++ code:

static void __Block_byref_id_object_copy_131(void *dst, void *src) {
 _Block_object_assign((char*)dst + 40, *(void * *) ((char*)src + 40), 131);
}
static void __Block_byref_id_object_dispose_131(void *src) {
 _Block_object_dispose(*(void * *) ((char*)src + 40), 131);
}

struct __Block_byref_obj_0 {
  void *__isa;
__Block_byref_obj_0 *__forwarding;
 int __flags;
 int __size;
 void (*__Block_byref_id_object_copy)(void*, void*);
 void (*__Block_byref_id_object_dispose)(void*);
 NSObject *obj;
};

struct __func_block_impl_0 {
  struct __block_impl impl;
  struct __func_block_desc_0* Desc;
  __Block_byref_obj_0 *obj; // by ref
  __func_block_impl_0(void *fp, struct __func_block_desc_0 *desc, __Block_byref_obj_0 *_obj, int flags=0) : obj(_obj->__forwarding) {
    impl.isa = &_NSConcreteStackBlock;
    impl.Flags = flags;
    impl.FuncPtr = fp;
    Desc = desc;
  }
};
static void __func_block_func_0(struct __func_block_impl_0 *__cself) {
  __Block_byref_obj_0 *obj = __cself->obj; // bound by ref

        (obj->__forwarding->obj) = __null;
    }
static void __func_block_copy_0(struct __func_block_impl_0*dst, struct __func_block_impl_0*src) {_Block_object_assign((void*)&dst->obj, (void*)src->obj, 8/*BLOCK_FIELD_IS_BYREF*/);}

static void __func_block_dispose_0(struct __func_block_impl_0*src) {_Block_object_dispose((void*)src->obj, 8/*BLOCK_FIELD_IS_BYREF*/);}

static struct __func_block_desc_0 {
  size_t reserved;
  size_t Block_size;
  void (*copy)(struct __func_block_impl_0*, struct __func_block_impl_0*);
  void (*dispose)(struct __func_block_impl_0*);
} __func_block_desc_0_DATA = { 0, sizeof(struct __func_block_impl_0), __func_block_copy_0, __func_block_dispose_0};
void func() {
    __attribute__((__blocks__(byref))) __Block_byref_obj_0 obj = {(void*)0,(__Block_byref_obj_0 *)&obj, 33554432, sizeof(__Block_byref_obj_0), __Block_byref_id_object_copy_131, __Block_byref_id_object_dispose_131, ((NSObject *(*)(id, SEL))(void *)objc_msgSend)((id)((NSObject *(*)(id, SEL))(void *)objc_msgSend)((id)objc_getClass("NSObject"), sel_registerName("alloc")), sel_registerName("init"))};
    void (*blk)(void) = ((void (*)())&__func_block_impl_0((void *)__func_block_func_0, &__func_block_desc_0_DATA, (__Block_byref_obj_0 *)&obj, 570425344));
}

NOTE: 33554432 is BLOCK_HAS_COPY_DISPOSE and 570425344 is BLOCK_HAS_COPY_DISPOSE|BLOCK_HAS_DESCRIPTOR.

When block is copied, __func_block_copy_0 is called to handle the variables it captured, in this case it makes a copy of (__Block_byref_obj_0)obj, changes obj->forwarding to the copy __Block_byref_obj_0 and so on, all of these operations happen in _Block_object_assign((void*)&dst->obj, (void*)src->obj, 8/*BLOCK_FIELD_IS_BYREF*/);.

The source code of _Block_object_assign:

void _Block_object_assign(void *destAddr, const void *object, const int flags) {
    //printf("_Block_object_assign(*%p, %p, %x)\n", destAddr, object, flags);
    if ((flags & BLOCK_BYREF_CALLER) == BLOCK_BYREF_CALLER) {
        if ((flags & BLOCK_FIELD_IS_WEAK) == BLOCK_FIELD_IS_WEAK) {
            _Block_assign_weak(object, destAddr);
        }
        else {
            // do *not* retain or *copy* __block variables whatever they are
            _Block_assign((void *)object, destAddr);
        }
    }
    else if ((flags & BLOCK_FIELD_IS_BYREF) == BLOCK_FIELD_IS_BYREF)  {
        // copying a __block reference from the stack Block to the heap
        // flags will indicate if it holds a __weak reference and needs a special isa
        _Block_byref_assign_copy(destAddr, object, flags);
    }
    // (this test must be before next one)
    else if ((flags & BLOCK_FIELD_IS_BLOCK) == BLOCK_FIELD_IS_BLOCK) {
        // copying a Block declared variable from the stack Block to the heap
        _Block_assign(_Block_copy_internal(object, flags), destAddr);
    }
    // (this test must be after previous one)
    else if ((flags & BLOCK_FIELD_IS_OBJECT) == BLOCK_FIELD_IS_OBJECT) {
        //printf("retaining object at %p\n", object);
        _Block_retain_object(object);
        //printf("done retaining object at %p\n", object);
        _Block_assign((void *)object, destAddr);
    }
}


static void _Block_byref_assign_copy(void *dest, const void *arg, const int flags) {
    struct Block_byref **destp = (struct Block_byref **)dest;
    struct Block_byref *src = (struct Block_byref *)arg;

    //printf("_Block_byref_assign_copy called, byref destp %p, src %p, flags %x\n", destp, src, flags);
    //printf("src dump: %s\n", _Block_byref_dump(src));
    if (src->forwarding->flags & BLOCK_IS_GC) {
        ;   // don't need to do any more work
    }
    else if ((src->forwarding->flags & BLOCK_REFCOUNT_MASK) == 0) {
        //printf("making copy\n");
        // src points to stack
        bool isWeak = ((flags & (BLOCK_FIELD_IS_BYREF|BLOCK_FIELD_IS_WEAK)) == (BLOCK_FIELD_IS_BYREF|BLOCK_FIELD_IS_WEAK));
        // if its weak ask for an object (only matters under GC)
        struct Block_byref *copy = (struct Block_byref *)_Block_allocator(src->size, false, isWeak);
        copy->flags = src->flags | _Byref_flag_initial_value; // non-GC one for caller, one for stack
        copy->forwarding = copy; // patch heap copy to point to itself (skip write-barrier)
        src->forwarding = copy;  // patch stack to point to heap copy
        copy->size = src->size;
        if (isWeak) {
            copy->isa = &_NSConcreteWeakBlockVariable;  // mark isa field so it gets weak scanning
        }
        if (src->flags & BLOCK_HAS_COPY_DISPOSE) {
            // Trust copy helper to copy everything of interest
            // If more than one field shows up in a byref block this is wrong XXX
            copy->byref_keep = src->byref_keep;
            copy->byref_destroy = src->byref_destroy;
            (*src->byref_keep)(copy, src);
        }
        else {
            // just bits.  Blast 'em using _Block_memmove in case they're __strong
            _Block_memmove(
                (void *)&copy->byref_keep,
                (void *)&src->byref_keep,
                src->size - sizeof(struct Block_byref_header));
        }
    }
    // already copied to heap
    else if ((src->forwarding->flags & BLOCK_NEEDS_FREE) == BLOCK_NEEDS_FREE) {
        latching_incr_int(&src->forwarding->flags);
    }
    // assign byref data block pointer into new Block
    _Block_assign(src->forwarding, (void **)destp);
}

As flag is BLOCK_FIELD_IS_BYREF, so the branch goes to _Block_byref_assign_copy, this function malloc memory for a copy of __Block_byref_obj_0 and do some assignments, finally it will call (*src->byref_keep)(copy, src) which points to __Block_byref_id_object_copy_131, as we can see in this function, only one line of code:

_Block_object_assign((char*)dst + 40, *(void * *) ((char*)src + 40), 131);, 131 is BLOCK_FIELD_IS_OBJECT|BLOCK_BYREF_CALLER, "(char*)dst+40" is the address of BLOCK_FIELD_IS_BYREF(copy), as a result it will call _Block_assign((void *)object, destAddr); and this function just do a assignment *destAddr = object;, no retain!!!

I believe obj should be retained but the source code seems that it doesn't retain it. I'm really confusing.

I got the source code of Blocks Runtime here, you can fetch it using svn co http://llvm.org/svn/llvm-project/compiler-rt/trunk compiler-rt.

Update 1:

static void _Block_assign_default(void *value, void **destptr) {
    *destptr = value;
}
static void (*_Block_assign)(void *value, void **destptr) = _Block_assign_default;

Answer 1

Yes and no. It doesn't directly "retain" it. I'll explain.

obj is a __block variable. As such, the block holds, in a way, a "reference" to the variable obj, and not a copy of it as it would other variables.

You say that blocks "retain" captured object pointer variables, and that's because blocks keep internal copies of captured regular (non-__block) variables, and keeping a new copy of a strong object pointer variable requires retaining it.

However, in this case, because it is __block, there is just one copy of the variable. The block (or blocks as it may be) and the original scope of the variable all share access to the same copy of the variable. The block doesn't have an independent copy of the variable. So why would the block retain the object it points to? (And imagine if blocks retained it, and one block changes the object that the variable points to, how would all the other blocks know to release the old object and retain the new one?)

(Note that __block variables start out on the stack and are moved to the heap, similar to blocks, but this optimization is not important to the memory management discussion since there will always be only one active copy of the variable.)

Another way of thinking about it is that __block variable really behave as if they were wrapped in some kind of transparent "holder" object to achieve the shared state. The "holder" object holds the wrapped variable as an internal field, and it retains the wrapped variable if it is of object-pointer type. However, the block (or blocks) that use this variable only hold a reference to the "holder" object, and not to the wrapped variable itself. All accesses to the variable are indirectly through references to the "holder" object. Therefore, when blocks are copied, they will retain the "holder" object, but not the variable inside. When all blocks that use this __block variable are deallocated, then there are no references to the "holder" object anymore, and the destructor of the "holder" object will in turn release the variable inside if it is of object-pointer type.

So the reference graph is like:

block --> holder object --> NSObject object

The block does indirectly retain the object, but not directly.

---

Update: So it seems like you are wondering why, if the "holder object" treats the underlying pointer variable as a strong reference, you don't see the "holder object" retain and release the underlying variable in __Block_byref_id_object_copy_131 and __Block_byref_id_object_dispose_131, respectively.

Actually, if you compile the code, you would see something different. Running clang -S -emit-llvm -fobjc-arc Block.m, you get textual LLVM IR, which includes the following copy and dispose helpers of the "holder object":

; Function Attrs: ssp uwtable
define internal void @__Block_byref_object_copy_(i8*, i8*) #0 {
  %3 = alloca i8*, align 8
  %4 = alloca i8*, align 8
  store i8* %0, i8** %3, align 8
  store i8* %1, i8** %4, align 8
  %5 = load i8*, i8** %3, align 8
  %6 = bitcast i8* %5 to %struct.__block_byref_obj*
  %7 = getelementptr inbounds %struct.__block_byref_obj, %struct.__block_byref_obj* %6, i32 0, i32 6
  %8 = load i8*, i8** %4, align 8
  %9 = bitcast i8* %8 to %struct.__block_byref_obj*
  %10 = getelementptr inbounds %struct.__block_byref_obj, %struct.__block_byref_obj* %9, i32 0, i32 6
  %11 = load %0*, %0** %10, align 8
  store %0* null, %0** %7, align 8
  %12 = bitcast %0** %7 to i8**
  %13 = bitcast %0* %11 to i8*
  call void @objc_storeStrong(i8** %12, i8* %13) #2
  %14 = bitcast %0** %10 to i8**
  call void @objc_storeStrong(i8** %14, i8* null) #2
  ret void
}

; Function Attrs: ssp uwtable
define internal void @__Block_byref_object_dispose_(i8*) #0 {
  %2 = alloca i8*, align 8
  store i8* %0, i8** %2, align 8
  %3 = load i8*, i8** %2, align 8
  %4 = bitcast i8* %3 to %struct.__block_byref_obj*
  %5 = getelementptr inbounds %struct.__block_byref_obj, %struct.__block_byref_obj* %4, i32 0, i32 6
  %6 = bitcast %0** %5 to i8**
  call void @objc_storeStrong(i8** %6, i8* null) #2
  ret void
}

In the __Block_byref_object_copy_ function, it performs an objc_storeStrong to assign the value of the variable in the stack version of the holder object strongly (i.e. with retain) to the variable in the heap version of the holder object, and the performs another objc_storeStrong to assign nil strongly (i.e. releasing what's there before) to the variable in the stack version of the holder object. In essence, it's what the following code would do in ARC:

heap_holder->var = stack_holder->var;
stack_holder->var = nil;

In the __Block_byref_object_dispose_ function, it performs an objc_storeStrong to assign nil strongly (i.e. releasing what's there before) to the variable in the heap version of the holder object. In essence, it's what the following code would do in ARC:

heap_holder->var = nil;

This is very different from what you got in the code generated by the "rewriter". I am guessing that the rewriter probably does not take ARC into account -- in other words, it performs an MRC -> MRC rewrite. If this were an MRC -> MRC rewrite, the generated code would indeed be correct, because __block variables are never retained in MRC, even if they are of object-pointer type. (There is other evidence that this is not a proper ARC -> ARC rewrite. For example, the calls to alloc and init are simply rewritten to calls to objc_msgSend. But alloc is a ns_returns_retained method in ARC, whereas objc_msgSend is not ns_returns_retained, so translating the former to the latter would cause a mismatch in retains that they do not remedy. But if this were an MRC -> MRC rewrite, this would be fine, as the user is responsible for explicitly making calls to retain/release.)

Indeed, if you compile it again without ARC, with clang -S -emit-llvm Block.m, you will see that the __Block_byref_object_copy_ and __Block_byref_object_dispose_ functions do use _Block_object_assign and _Block_object_dispose, respectively, matching what your rewritten code shows.

If we look at the Clang source for blocks code generation, at the part which builds the byref helpers, CodeGenFunction::buildByrefHelpers, there is an if in there (line 1970) where it checks if the variable has a "lifetime" (which I think means it's a managed type under ARC), and if so, it builds using ARCWeakByrefHelpers or ARCStrongByrefHelpers or ARCStrongBlockByrefHelpers; but if it doesn't have a lifetime, it builds using ObjectByrefHelpers. Looking at ARCStrongByrefHelpers for example, we see that it emits two store-strongs in the copy helper, and a destroy-strong in the dispose helper, which is what we see in the ARC compiled code. On the other hand, in ObjectByrefHelpers, we see that it emits block-object-assign in the copy helper and block-release in the dispose helper, which is what we see in the MRC compiled code.

But if you look at the source of the Objective-C rewriter, the method for generating the byref copy and dispose helpers, RewriteModernObjC::SynthesizeByrefCopyDestroyHelper, always generates _Block_object_assign and _Block_object_dispose. So this matches the hypothesis that the rewriter only does an MRC rewrite, though I cannot find any documentation on the rewriter so I cannot say whether this is by design or not. There is an answer to another question that indicates that ARC operates at the code-gen level, which may perhaps be a reason why the source-to-source rewriter does not consider it(?).

Answer 2

Thanks to the advice of @CRD and @newacct, I divide into assembly code and find some clue. I will post the assembly code here and do some analysis.

The assembly code target is armv7, so the long and pointer take 4 bytes for your information.

My first target is to find the __Block_byref_id_object_copy function which is referenced in my question and used to handle NSObject *obj where the block is copied to heap, it locates in __Block_byref_obj_0, let's review the struct.

Let's use "holder object" instead of __Block_byref_obj_0 for simplification.

struct __Block_byref_obj_0 {
  void *__isa;                      // address 0
__Block_byref_obj_0 *__forwarding;  // address 4
 int __flags;                       // address 8
 int __size;                        // address 12
 void (*__Block_byref_id_object_copy)(void*, void*); // address 16
 void (*__Block_byref_id_object_dispose)(void*);     // address 20
 NSObject *obj;                     // address 24
};

As the comments indicate, __Block_byref_id_object_copy is at the address of holder object + 16 and obj is at the address of holder object + 24.

Now I post part of the main function in Block.m.

@ BB#0:
    push    {r4, r5, r6, r7, lr}
    add r7, sp, #12
    push.w  {r8, r10, r11}
    sub.w   r4, sp, #64
    bfc r4, #0, #4
    mov sp, r4
    vst1.64 {d8, d9, d10, d11}, [r4:128]!
    vst1.64 {d12, d13, d14, d15}, [r4:128]
    sub sp, #160
    movs    r0, #0
    .loc    1 27 23 prologue_end
Ltmp3:
    str r0, [sp, #80]
    add r1, sp, #80
    str r1, [sp, #84]
    mov.w   r2, #838860800
    str r2, [sp, #88]
    movs    r2, #28
    str r2, [sp, #92]
    movw    r2, :lower16:(___Block_byref_object_copy_-(LPC0_2+4))
    movt    r2, :upper16:(___Block_byref_object_copy_-(LPC0_2+4))
LPC0_2:
    add r2, pc
    str r2, [sp, #96]
    movw    r2, :lower16:(___Block_byref_object_dispose_-(LPC0_3+4))
    movt    r2, :upper16:(___Block_byref_object_dispose_-(LPC0_3+4))
LPC0_3:
    add r2, pc
    str r2, [sp, #100]
    add.w   r2, r1, #24
    .loc    1 27 30 is_stmt 0       
    movw    r3, :lower16:(L_OBJC_CLASSLIST_REFERENCES_$_-(LPC0_4+4))
    movt    r3, :upper16:(L_OBJC_CLASSLIST_REFERENCES_$_-(LPC0_4+4))
LPC0_4:
    add r3, pc
    ldr r3, [r3]
    movw    r9, :lower16:(L_OBJC_SELECTOR_REFERENCES_-(LPC0_5+4))
    movt    r9, :upper16:(L_OBJC_SELECTOR_REFERENCES_-(LPC0_5+4))
LPC0_5:
    add r9, pc
    ldr.w   r9, [r9]
    str r0, [sp, #40]           @ 4-byte Spill
    mov r0, r3
    str r1, [sp, #36]           @ 4-byte Spill
    mov r1, r9
    str r2, [sp, #32]           @ 4-byte Spill
    blx _objc_msgSend
    .loc    1 27 29     
    movw    r1, :lower16:(L_OBJC_SELECTOR_REFERENCES_.2-(LPC0_6+4))
    movt    r1, :upper16:(L_OBJC_SELECTOR_REFERENCES_.2-(LPC0_6+4))
LPC0_6:
    add r1, pc
    ldr r1, [r1]
    blx _objc_msgSend
    .loc    1 27 23      
    str r0, [sp, #104]
    .loc    1 27 5       
    ldr r0, [sp, #32]           @ 4-byte Reload
    .loc    1 28 25 is_stmt 1 
    movw    r1, :lower16:(L__NSConcreteStackBlock$non_lazy_ptr-(LPC0_7+4))
    movt    r1, :upper16:(L__NSConcreteStackBlock$non_lazy_ptr-(LPC0_7+4))
LPC0_7:
    add r1, pc
    ldr r1, [r1]
    str r1, [sp, #52]
    mov.w   r1, #-1040187392
    str r1, [sp, #56]
    ldr r1, [sp, #40]           @ 4-byte Reload
    str r1, [sp, #60]
    movw    r2, :lower16:(___func_block_invoke-(LPC0_8+4))
    movt    r2, :upper16:(___func_block_invoke-(LPC0_8+4))
LPC0_8:
    add r2, pc
    str r2, [sp, #64]
    movw    r2, :lower16:(___block_descriptor_tmp-(LPC0_9+4))
    movt    r2, :upper16:(___block_descriptor_tmp-(LPC0_9+4))

Look at these code :

    movw    r1, :lower16:(L_OBJC_SELECTOR_REFERENCES_.2-(LPC0_6+4))
    movt    r1, :upper16:(L_OBJC_SELECTOR_REFERENCES_.2-(LPC0_6+4))
LPC0_6:
    add r1, pc
    ldr r1, [r1]
    blx _objc_msgSend
    .loc    1 27 23      
    str r0, [sp, #104]

L_OBJC_SELECTOR_REFERENCES_.2 is init method, and the return object is in r0, r0 is then stored in sp+104, so I know __Block_byref_id_object_copy must be stored at sp+96, it's ___Block_byref_object_copy_.

Now let's pay attention to ___Block_byref_object_copy_.

push    {r7, lr}
mov r7, sp
sub sp, #12
movs    r2, #0
str r0, [sp, #8]
str r1, [sp, #4]
ldr r0, [sp, #8]
mov r1, r0
adds    r1, #24
ldr r3, [sp, #4]
mov r9, r3
add.w   r9, r9, #24
ldr r3, [r3, #24]
str r2, [r0, #24]
mov r0, r1
mov r1, r3
str.w   r9, [sp]
bl  _objc_storeStrong
movs    r1, #0
ldr r0, [sp]
bl  _objc_storeStrong
add sp, #12
pop {r7, pc}

To remind you that static void __Block_byref_id_object_copy_131(void *dst, void *src) is the same as ___Block_byref_object_copy_, I don't know what compiler does behind but it indeed change the function name, never mind, what I need to know it the first parameter is the address of destination holder object and the second parameter is the address of the source holder object.

So in ___Block_byref_object_copy_, r0 stores the address of destination holder, r1 stores the address of source holder.

str r0, [sp, #8]
str r1, [sp, #4]
ldr r0, [sp, #8]
mov r1, r0

It stores r0 to sp+8 and r1 to sp+4, move r0 to r1. So r1=r0=address of dst holder.

adds    r1, #24
ldr r3, [sp, #4]
mov r9, r3
add.w   r9, r9, #24

Then it adds r1+24 to r1, so r1 stores the address of obj in dst holder., loads the value at [sp+4] to r3, so r3 stores the address of src holder, move r3 to r9, adds r9+24 to r9, so r9 has the address of obj in src holder.

ldr r3, [r3, #24]
str r2, [r0, #24]
mov r0, r1
mov r1, r3
str.w   r9, [sp]

It loads the value in [r3+24] to r3, so r3 stores the obj in src holder, store r2 to r0+24, r2 is zero, so the obj in dst holder is NULL. Then move r1 to r0, r0 has the address of obj in dst holder, move r3 to r1, so r1 stores the obj in src holder.

bl  _objc_storeStrong
movs    r1, #0
ldr r0, [sp]
bl  _objc_storeStrong

To know what is objc_storeStrong, check here

When bl _objc_storeStrong calls, r0 is the address of obj in dst holder and r1 is the obj in src holder. objc_storeStrong will retain the obj in src holder and assign it to the obj in dst holder.

Then it assign 0 to r1, load [sp] to r0, [sp] stores the address of obj in src holder.

bl _objc_storeStrong is doing objc_storeStrong(&obj_src_holder, nil), so obj_src_holder would be sent release method and assigned to nil.

Conclude: I don't know why it release the obj in source holder and assign to nil, shouldn't it be released at the end of func() ?

However, the obj in destination holder keep a strong reference to the NSObject *obj indeed, so this should be an reasonable answer I think.

Answer 3

I think what you are missing in your analysis is that you are not seeing the calls inserted for ARC, rather you are seeing language level assignments.

In __Block_byref_obj_0 you show the field obj as:

NSObject *obj;

without any explicit strong ownership qualifier. When I ran similar tests Clang output:

NSObject *__strong obj;

including an explicit ownership qualifier.

If you look at the assembly level code you will see there are calls inserted to ARC memory management routines in various places.

So what you are seeing as a simple assignment may in fact be compiled to a strong store - i.e. release any existing reference, retain the new one. This is of course exactly the same as in the original Objective-C, you "read in" the ARC semantics as an integral part of the language semantics.

HTH