Continuing to look at OpenBIOS. Here is a classic integer overflow leading to a heap based buffer overflow.
nhdr->n_descsz is 32-bits. So we on x86 (32-bit), if we make it UINT_MAX, we cause ob_malloc to allocate 0 bytes. This returns a pointer. The memcpy causes memory corruption. Even if ob_calloc returned NULL, there is no error checking.
if (nhdr->n_namesz==sizeof(ELF_NOTE_BOOT)
&& memcmp(name, ELF_NOTE_BOOT, sizeof(ELF_NOTE_BOOT))==0) {
if (nhdr->n_type == EIN_PROGRAM_NAME) {
image_name = ob_calloc(1, nhdr->n_descsz + 1);
memcpy(image_name, desc, nhdr->n_descsz);
}
And to see what ob_calloc does:
static void *ob_calloc(size_t nmemb, size_t size)
{
size_t alloc_size = nmemb * size;
void *mem;
if (alloc_size < nmemb || alloc_size < size) {
printf("calloc overflow: %u, %u\n", nmemb, size);
return NULL;
}
mem = malloc(alloc_size);
memset(mem, 0, alloc_size);
return mem;
}
And to verify that malloc(0) returns a pointer, we have:
void *malloc(int size)
{
void *ret=(void *)0;
if(memsize>=size) {
memsize-=size;
ret=memptr;
memptr = (void *)((unsigned long)memptr + size);
}
return ret;
}