Using VMM as standalone hypervisor

abhichavali · Mar 18, 2025

I understand VMM is tightly coupled with Bhyve, but for an experiment I wanted to get a small real mode program running using only vmm (a replica of nvmm). This was my best attempt. I made sure to set the segment registers appropriately, but gdb tells me I'm calling vm_run improperly. Any help is appreciated.

Code:

#include <vmmapi.h>
#include <stdint.h>
#include <stdio.h>
#include <memory.h>
#include <stdlib.h>
#include <assert.h>
#include <sys/mman.h>
#include <err.h>
#include <errno.h>


/* Adder Program

   Uses vmm to take two arguments from command line, and add them using
   assembly instructions. Runs in 16 Bit real mode

*/
#define USER_PAGE_SIZE 4096 * 1024
#define MEMSIZE USER_PAGE_SIZE * 1

const uint8_t instr[] = {
        0x01, 0xC3,
        0xF4
};

int main(int argc, char* argv[]) {
        struct vmctx* machine_ctx;
        struct vcpu* cpu;
        int e, rax, rbx;
        vm_paddr_t gpa = 0x10000;
     
        struct vm_exit vme;
        struct vm_run vmrun;
        int error;
        uint64_t rc;
        enum vm_exitcode exitcode;
        cpuset_t active_cpus, dmask;
     
        rax = atoi(argv[1]);
        rbx = atoi(argv[2]);

        // Close if active
  
        const char vm_name[] = "adder";

        machine_ctx = vm_open(vm_name);
        if(machine_ctx) {
                vm_close(machine_ctx);
                vm_destroy(machine_ctx);
        }
        // Create machine
        if((e = vm_create(vm_name)) != 0)
                errx(EXIT_FAILURE, "Could not create vm %s\n", vm_name);

        machine_ctx = vm_open(vm_name);
     
        // Setup Memory
        e = vm_setup_memory(machine_ctx, MEMSIZE, VM_MMAP_ALL);
        assert(e == 0);
     
        void* v = vm_map_gpa(machine_ctx, gpa, sizeof(instr));
        memcpy(v, instr, sizeof(instr));
     
        // Initialize vCPU
        cpu = vm_vcpu_open(machine_ctx, 0);

        e = vm_active_cpus(machine_ctx, &active_cpus);

        memset(&vmrun, 0, sizeof(vmrun))
        vmrun.vm_exit = NULL;

        // Set Registers
        e = vm_set_register(cpu, VM_REG_GUEST_RAX, rax);
        assert(e == 0);
        e = vm_set_register(cpu, VM_REG_GUEST_RBX, rbx);
        assert(e == 0);
        e = vm_set_register(cpu, VM_REG_GUEST_RIP, 0x0);
        assert(e == 0);
        e = vm_set_register(cpu, VM_REG_GUEST_CS, 0x1000);
        assert(e == 0);
        e = vm_set_register(cpu, VM_REG_GUEST_RFLAGS, 0x2);  // Interrupt Flag clear, reserved bit set

        // Set up segment registers for real mode
        e = vm_set_register(cpu, VM_REG_GUEST_DS, 0);
        e = vm_set_register(cpu, VM_REG_GUEST_ES, 0);
        e = vm_set_register(cpu, VM_REG_GUEST_FS, 0);
        e = vm_set_register(cpu, VM_REG_GUEST_GS, 0);
        e = vm_set_register(cpu, VM_REG_GUEST_SS, 0);

        // Set CR0 for real mode
        e = vm_set_register(cpu, VM_REG_GUEST_CR0, 0);

        // Execution Loop
        while(1) {
                e = vm_run(cpu, &vmrun);
                if (e < 0) {
                        perror("vm_run failed");
                        printf("Error code: %d\n", errno);
                        goto out;
                }
                switch(vme.exitcode) {
                case VM_EXITCODE_HLT:
                        printf("Encountered HLT\n");
                        rc = vm_get_register(cpu, VM_REG_GUEST_RBX, &rc);
                        printf("Value: %lu\n", rc);
                        goto out;
                }
        }
 out:
        vm_close(machine_ctx);
        vm_destroy(machine_ctx);
        return e;
}

atax1a · Thursday at 3:40 AM

You're setting null selectors in the segment registers, and you're also neglecting to set the stack pointer. You're also starting the instruction pointer off at 0, which is where the IDT should reside.

atax1a · Thursday at 7:37 PM

rereading this after a good night's sleep, you're indeed loading the code segment register and so that should be okay, but it's probably unhappy about the null selectors for the stack segment and the unset stack pointer.

abhichavali · Saturday at 1:04 AM

I modified the stack pointer and segment but vm_run still isn't working properly. I suspect it might be in me calling vm_run incorrectly, but I believe I'm doing it the way Bhyve is:

...
memset(&vmrun, 0, sizeof(vmrun));
vmrun.vm_exit = &vme;
vmrun.cpuset = &dmask;
vmrun.cpusetsize = sizeof(dmask);
....
e = vm_set_register(cpu, VM_REG_GUEST_RFLAGS, 0x2); // Interrupt Flag clear, reserved bit set

// Set up segment registers for real mode
e = vm_set_register(cpu, VM_REG_GUEST_DS, 0x1000);
e = vm_set_register(cpu, VM_REG_GUEST_ES, 0x1000);
e = vm_set_register(cpu, VM_REG_GUEST_FS, 0x1000);
e = vm_set_register(cpu, VM_REG_GUEST_GS, 0x1000);
e = vm_set_register(cpu, VM_REG_GUEST_SS, 0x1000);

// Set stack pointer
e = vm_set_register(cpu, VM_REG_GUEST_RSP, 0x7C00); // Example stack pointer value

// Set CR0 for real mode
e = vm_set_register(cpu, VM_REG_GUEST_CR0, 0);
...
// Execution Loop
while(1) {
e = vm_run(cpu, &vmrun);
if (e != 0) {
perror("vm_run failed");
printf("Error code: %d\n", errno);
goto out;
}
switch(vme.exitcode) {
case VM_EXITCODE_HLT:
printf("Encountered HLT\n");
rc = vm_get_register(cpu, VM_REG_GUEST_RBX, &rc);
printf("Value: %lu\n", rc);
goto out;
}
}

Using VMM as standalone hypervisor

abhichavali

atax1a

atax1a

abhichavali