kvm-all.c 52.2 KB
Newer Older
1 2 3 4
/*
 * QEMU KVM support
 *
 * Copyright IBM, Corp. 2008
5
 *           Red Hat, Inc. 2008
6 7 8
 *
 * Authors:
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Glauber Costa     <gcosta@redhat.com>
10 11 12 13 14 15 16 17 18
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */

#include <sys/types.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
19
#include <stdarg.h>
20 21 22 23

#include <linux/kvm.h>

#include "qemu-common.h"
24 25 26
#include "qemu/atomic.h"
#include "qemu/option.h"
#include "qemu/config-file.h"
27
#include "sysemu/sysemu.h"
Jan Kiszka's avatar
Jan Kiszka committed
28
#include "hw/hw.h"
29
#include "hw/pci/msi.h"
30
#include "exec/gdbstub.h"
31
#include "sysemu/kvm.h"
32
#include "qemu/bswap.h"
33
#include "exec/memory.h"
34
#include "exec/ram_addr.h"
35
#include "exec/address-spaces.h"
36
#include "qemu/event_notifier.h"
37
#include "trace.h"
38

39 40 41 42 43
/* This check must be after config-host.h is included */
#ifdef CONFIG_EVENTFD
#include <sys/eventfd.h>
#endif

44 45 46 47
#ifdef CONFIG_VALGRIND_H
#include <valgrind/memcheck.h>
#endif

48
/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
aliguori's avatar
aliguori committed
49 50
#define PAGE_SIZE TARGET_PAGE_SIZE

51 52 53
//#define DEBUG_KVM

#ifdef DEBUG_KVM
54
#define DPRINTF(fmt, ...) \
55 56
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
#else
57
#define DPRINTF(fmt, ...) \
58 59 60
    do { } while (0)
#endif

61 62
#define KVM_MSI_HASHTAB_SIZE    256

aliguori's avatar
aliguori committed
63 64
typedef struct KVMSlot
{
65
    hwaddr start_addr;
66
    ram_addr_t memory_size;
67
    void *ram;
aliguori's avatar
aliguori committed
68 69 70
    int slot;
    int flags;
} KVMSlot;
71

72 73
typedef struct kvm_dirty_log KVMDirtyLog;

74 75
struct KVMState
{
76 77
    KVMSlot *slots;
    int nr_slots;
78 79
    int fd;
    int vmfd;
aliguori's avatar
aliguori committed
80
    int coalesced_mmio;
81
    struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
82
    bool coalesced_flush_in_progress;
83
    int broken_set_mem_region;
84
    int migration_log;
85
    int vcpu_events;
86
    int robust_singlestep;
87
    int debugregs;
88 89 90
#ifdef KVM_CAP_SET_GUEST_DEBUG
    struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
#endif
91
    int pit_state2;
92
    int xsave, xcrs;
93
    int many_ioeventfds;
94
    int intx_set_mask;
95 96 97
    /* The man page (and posix) say ioctl numbers are signed int, but
     * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
     * unsigned, and treating them as signed here can break things */
98
    unsigned irq_set_ioctl;
99 100 101 102
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing *irq_routes;
    int nr_allocated_irq_routes;
    uint32_t *used_gsi_bitmap;
103
    unsigned int gsi_count;
104
    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
105
    bool direct_msi;
106
#endif
107 108
};

109
KVMState *kvm_state;
110
bool kvm_kernel_irqchip;
111
bool kvm_async_interrupts_allowed;
112
bool kvm_halt_in_kernel_allowed;
113
bool kvm_irqfds_allowed;
114
bool kvm_msi_via_irqfd_allowed;
115
bool kvm_gsi_routing_allowed;
116
bool kvm_gsi_direct_mapping;
117
bool kvm_allowed;
118
bool kvm_readonly_mem_allowed;
119

120 121 122 123 124 125
static const KVMCapabilityInfo kvm_required_capabilites[] = {
    KVM_CAP_INFO(USER_MEMORY),
    KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
    KVM_CAP_LAST_INFO
};

126 127 128 129
static KVMSlot *kvm_alloc_slot(KVMState *s)
{
    int i;

130
    for (i = 0; i < s->nr_slots; i++) {
131
        if (s->slots[i].memory_size == 0) {
132
            return &s->slots[i];
133
        }
134 135
    }

136 137 138 139 140
    fprintf(stderr, "%s: no free slot available\n", __func__);
    abort();
}

static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
141 142
                                         hwaddr start_addr,
                                         hwaddr end_addr)
143 144 145
{
    int i;

146
    for (i = 0; i < s->nr_slots; i++) {
147 148 149 150 151 152 153 154
        KVMSlot *mem = &s->slots[i];

        if (start_addr == mem->start_addr &&
            end_addr == mem->start_addr + mem->memory_size) {
            return mem;
        }
    }

155 156 157
    return NULL;
}

158 159 160 161
/*
 * Find overlapping slot with lowest start address
 */
static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
162 163
                                            hwaddr start_addr,
                                            hwaddr end_addr)
164
{
165
    KVMSlot *found = NULL;
166 167
    int i;

168
    for (i = 0; i < s->nr_slots; i++) {
169 170
        KVMSlot *mem = &s->slots[i];

171 172 173 174 175 176 177 178 179
        if (mem->memory_size == 0 ||
            (found && found->start_addr < mem->start_addr)) {
            continue;
        }

        if (end_addr > mem->start_addr &&
            start_addr < mem->start_addr + mem->memory_size) {
            found = mem;
        }
180 181
    }

182
    return found;
183 184
}

185
int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
186
                                       hwaddr *phys_addr)
187 188 189
{
    int i;

190
    for (i = 0; i < s->nr_slots; i++) {
191 192
        KVMSlot *mem = &s->slots[i];

193 194
        if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
            *phys_addr = mem->start_addr + (ram - mem->ram);
195 196 197 198 199 200 201
            return 1;
        }
    }

    return 0;
}

202 203 204 205 206 207
static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
{
    struct kvm_userspace_memory_region mem;

    mem.slot = slot->slot;
    mem.guest_phys_addr = slot->start_addr;
208
    mem.userspace_addr = (unsigned long)slot->ram;
209
    mem.flags = slot->flags;
210 211 212
    if (s->migration_log) {
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
    }
213 214

    if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
215 216 217 218 219 220
        /* Set the slot size to 0 before setting the slot to the desired
         * value. This is needed based on KVM commit 75d61fbc. */
        mem.memory_size = 0;
        kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
    }
    mem.memory_size = slot->memory_size;
221 222 223
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
}

224 225
static void kvm_reset_vcpu(void *opaque)
{
226
    CPUState *cpu = opaque;
227

228
    kvm_arch_reset_vcpu(cpu);
229
}
230

231
int kvm_init_vcpu(CPUState *cpu)
232 233 234 235 236
{
    KVMState *s = kvm_state;
    long mmap_size;
    int ret;

237
    DPRINTF("kvm_init_vcpu\n");
238

239
    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
240
    if (ret < 0) {
241
        DPRINTF("kvm_create_vcpu failed\n");
242 243 244
        goto err;
    }

245
    cpu->kvm_fd = ret;
246
    cpu->kvm_state = s;
247
    cpu->kvm_vcpu_dirty = true;
248 249 250

    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
    if (mmap_size < 0) {
251
        ret = mmap_size;
252
        DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
253 254 255
        goto err;
    }

256
    cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
257
                        cpu->kvm_fd, 0);
258
    if (cpu->kvm_run == MAP_FAILED) {
259
        ret = -errno;
260
        DPRINTF("mmap'ing vcpu state failed\n");
261 262 263
        goto err;
    }

264 265
    if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
        s->coalesced_mmio_ring =
266
            (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
267
    }
268

269
    ret = kvm_arch_init_vcpu(cpu);
270
    if (ret == 0) {
271 272
        qemu_register_reset(kvm_reset_vcpu, cpu);
        kvm_arch_reset_vcpu(cpu);
273
    }
274 275 276 277
err:
    return ret;
}

278 279 280
/*
 * dirty pages logging control
 */
281

282
static int kvm_mem_flags(KVMState *s, bool log_dirty, bool readonly)
283
{
284 285 286 287 288 289
    int flags = 0;
    flags = log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
    if (readonly && kvm_readonly_mem_allowed) {
        flags |= KVM_MEM_READONLY;
    }
    return flags;
290 291 292
}

static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
293 294
{
    KVMState *s = kvm_state;
295
    int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
296 297 298
    int old_flags;

    old_flags = mem->flags;
299

300
    flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty, false);
301 302
    mem->flags = flags;

303 304 305 306
    /* If nothing changed effectively, no need to issue ioctl */
    if (s->migration_log) {
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
    }
307

308
    if (flags == old_flags) {
309
        return 0;
310 311
    }

312 313 314
    return kvm_set_user_memory_region(s, mem);
}

315
static int kvm_dirty_pages_log_change(hwaddr phys_addr,
316 317 318 319 320 321 322 323
                                      ram_addr_t size, bool log_dirty)
{
    KVMState *s = kvm_state;
    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);

    if (mem == NULL)  {
        fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
                TARGET_FMT_plx "\n", __func__, phys_addr,
324
                (hwaddr)(phys_addr + size - 1));
325 326 327 328 329
        return -EINVAL;
    }
    return kvm_slot_dirty_pages_log_change(mem, log_dirty);
}

330 331
static void kvm_log_start(MemoryListener *listener,
                          MemoryRegionSection *section)
332
{
333 334 335
    int r;

    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
336
                                   int128_get64(section->size), true);
337 338 339
    if (r < 0) {
        abort();
    }
340 341
}

342 343
static void kvm_log_stop(MemoryListener *listener,
                          MemoryRegionSection *section)
344
{
345 346 347
    int r;

    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
348
                                   int128_get64(section->size), false);
349 350 351
    if (r < 0) {
        abort();
    }
352 353
}

354
static int kvm_set_migration_log(int enable)
355 356 357 358 359 360 361
{
    KVMState *s = kvm_state;
    KVMSlot *mem;
    int i, err;

    s->migration_log = enable;

362
    for (i = 0; i < s->nr_slots; i++) {
363 364
        mem = &s->slots[i];

365 366 367
        if (!mem->memory_size) {
            continue;
        }
368 369 370 371 372 373 374 375 376 377 378
        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
            continue;
        }
        err = kvm_set_user_memory_region(s, mem);
        if (err) {
            return err;
        }
    }
    return 0;
}

379
/* get kvm's dirty pages bitmap and update qemu's */
380 381
static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
                                         unsigned long *bitmap)
382
{
383
    ram_addr_t start = section->offset_within_region + section->mr->ram_addr;
384 385 386
    ram_addr_t pages = int128_get64(section->size) / getpagesize();

    cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
387
    return 0;
388 389
}

390 391
#define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))

392 393
/**
 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
394 395 396
 * This function updates qemu's dirty bitmap using
 * memory_region_set_dirty().  This means all bits are set
 * to dirty.
397
 *
398
 * @start_add: start of logged region.
399 400
 * @end_addr: end of logged region.
 */
401
static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
402 403
{
    KVMState *s = kvm_state;
404 405 406 407
    unsigned long size, allocated_size = 0;
    KVMDirtyLog d;
    KVMSlot *mem;
    int ret = 0;
408
    hwaddr start_addr = section->offset_within_address_space;
409
    hwaddr end_addr = start_addr + int128_get64(section->size);
410

411 412 413 414 415 416
    d.dirty_bitmap = NULL;
    while (start_addr < end_addr) {
        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
        if (mem == NULL) {
            break;
        }
417

418 419 420 421 422 423 424 425 426 427 428 429 430 431
        /* XXX bad kernel interface alert
         * For dirty bitmap, kernel allocates array of size aligned to
         * bits-per-long.  But for case when the kernel is 64bits and
         * the userspace is 32bits, userspace can't align to the same
         * bits-per-long, since sizeof(long) is different between kernel
         * and user space.  This way, userspace will provide buffer which
         * may be 4 bytes less than the kernel will use, resulting in
         * userspace memory corruption (which is not detectable by valgrind
         * too, in most cases).
         * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
         * a hope that sizeof(long) wont become >8 any time soon.
         */
        size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
                     /*HOST_LONG_BITS*/ 64) / 8;
432
        if (!d.dirty_bitmap) {
433
            d.dirty_bitmap = g_malloc(size);
434
        } else if (size > allocated_size) {
435
            d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
436 437 438
        }
        allocated_size = size;
        memset(d.dirty_bitmap, 0, allocated_size);
439

440
        d.slot = mem->slot;
441

442
        if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
443
            DPRINTF("ioctl failed %d\n", errno);
444 445 446
            ret = -1;
            break;
        }
447

448
        kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
449
        start_addr = mem->start_addr + mem->memory_size;
450
    }
451
    g_free(d.dirty_bitmap);
452 453

    return ret;
454 455
}

456 457
static void kvm_coalesce_mmio_region(MemoryListener *listener,
                                     MemoryRegionSection *secion,
458
                                     hwaddr start, hwaddr size)
aliguori's avatar
aliguori committed
459 460 461 462 463 464 465 466
{
    KVMState *s = kvm_state;

    if (s->coalesced_mmio) {
        struct kvm_coalesced_mmio_zone zone;

        zone.addr = start;
        zone.size = size;
467
        zone.pad = 0;
aliguori's avatar
aliguori committed
468

469
        (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
aliguori's avatar
aliguori committed
470 471 472
    }
}

473 474
static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
                                       MemoryRegionSection *secion,
475
                                       hwaddr start, hwaddr size)
aliguori's avatar
aliguori committed
476 477 478 479 480 481 482 483
{
    KVMState *s = kvm_state;

    if (s->coalesced_mmio) {
        struct kvm_coalesced_mmio_zone zone;

        zone.addr = start;
        zone.size = size;
484
        zone.pad = 0;
aliguori's avatar
aliguori committed
485

486
        (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
aliguori's avatar
aliguori committed
487 488 489
    }
}

490 491 492 493 494 495 496 497 498 499 500 501
int kvm_check_extension(KVMState *s, unsigned int extension)
{
    int ret;

    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
    if (ret < 0) {
        ret = 0;
    }

    return ret;
}

502
static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
503
                                  bool assign, uint32_t size, bool datamatch)
504 505 506 507
{
    int ret;
    struct kvm_ioeventfd iofd;

508
    iofd.datamatch = datamatch ? val : 0;
509 510
    iofd.addr = addr;
    iofd.len = size;
511
    iofd.flags = 0;
512 513 514 515 516 517
    iofd.fd = fd;

    if (!kvm_enabled()) {
        return -ENOSYS;
    }

518 519 520
    if (datamatch) {
        iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
    }
521 522 523 524 525 526 527 528 529 530 531 532 533
    if (!assign) {
        iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
    }

    ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);

    if (ret < 0) {
        return -errno;
    }

    return 0;
}

534
static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
535
                                 bool assign, uint32_t size, bool datamatch)
536 537
{
    struct kvm_ioeventfd kick = {
538
        .datamatch = datamatch ? val : 0,
539
        .addr = addr,
540
        .flags = KVM_IOEVENTFD_FLAG_PIO,
541
        .len = size,
542 543 544 545 546 547
        .fd = fd,
    };
    int r;
    if (!kvm_enabled()) {
        return -ENOSYS;
    }
548 549 550
    if (datamatch) {
        kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
    }
551 552 553 554 555 556 557 558 559 560 561
    if (!assign) {
        kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
    }
    r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
    if (r < 0) {
        return r;
    }
    return 0;
}


562 563
static int kvm_check_many_ioeventfds(void)
{
564 565 566 567 568
    /* Userspace can use ioeventfd for io notification.  This requires a host
     * that supports eventfd(2) and an I/O thread; since eventfd does not
     * support SIGIO it cannot interrupt the vcpu.
     *
     * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
569 570
     * can avoid creating too many ioeventfds.
     */
571
#if defined(CONFIG_EVENTFD)
572 573 574 575 576 577 578
    int ioeventfds[7];
    int i, ret = 0;
    for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
        ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
        if (ioeventfds[i] < 0) {
            break;
        }
579
        ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
580 581 582 583 584 585 586 587 588 589
        if (ret < 0) {
            close(ioeventfds[i]);
            break;
        }
    }

    /* Decide whether many devices are supported or not */
    ret = i == ARRAY_SIZE(ioeventfds);

    while (i-- > 0) {
590
        kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
591 592 593 594 595 596 597 598
        close(ioeventfds[i]);
    }
    return ret;
#else
    return 0;
#endif
}

599 600 601 602 603 604 605 606 607 608 609 610
static const KVMCapabilityInfo *
kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
{
    while (list->name) {
        if (!kvm_check_extension(s, list->value)) {
            return list;
        }
        list++;
    }
    return NULL;
}

611
static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
612 613 614 615
{
    KVMState *s = kvm_state;
    KVMSlot *mem, old;
    int err;
616 617
    MemoryRegion *mr = section->mr;
    bool log_dirty = memory_region_is_logging(mr);
618 619
    bool writeable = !mr->readonly && !mr->rom_device;
    bool readonly_flag = mr->readonly || memory_region_is_romd(mr);
620
    hwaddr start_addr = section->offset_within_address_space;
621
    ram_addr_t size = int128_get64(section->size);
622
    void *ram = NULL;
Avi Kivity's avatar
Avi Kivity committed
623
    unsigned delta;
624

625 626
    /* kvm works in page size chunks, but the function may be called
       with sub-page size and unaligned start address. */
Avi Kivity's avatar
Avi Kivity committed
627 628 629 630 631 632 633 634 635 636
    delta = TARGET_PAGE_ALIGN(size) - size;
    if (delta > size) {
        return;
    }
    start_addr += delta;
    size -= delta;
    size &= TARGET_PAGE_MASK;
    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
        return;
    }
637

638
    if (!memory_region_is_ram(mr)) {
639 640 641 642 643 644 645
        if (writeable || !kvm_readonly_mem_allowed) {
            return;
        } else if (!mr->romd_mode) {
            /* If the memory device is not in romd_mode, then we actually want
             * to remove the kvm memory slot so all accesses will trap. */
            add = false;
        }
646 647
    }

Avi Kivity's avatar
Avi Kivity committed
648
    ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
649

650 651 652 653 654 655
    while (1) {
        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
        if (!mem) {
            break;
        }

656
        if (add && start_addr >= mem->start_addr &&
657
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
658
            (ram - start_addr == mem->ram - mem->start_addr)) {
659
            /* The new slot fits into the existing one and comes with
660 661
             * identical parameters - update flags and done. */
            kvm_slot_dirty_pages_log_change(mem, log_dirty);
662 663 664 665 666
            return;
        }

        old = *mem;

667 668 669 670
        if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
            kvm_physical_sync_dirty_bitmap(section);
        }

671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
        /* unregister the overlapping slot */
        mem->memory_size = 0;
        err = kvm_set_user_memory_region(s, mem);
        if (err) {
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
                    __func__, strerror(-err));
            abort();
        }

        /* Workaround for older KVM versions: we can't join slots, even not by
         * unregistering the previous ones and then registering the larger
         * slot. We have to maintain the existing fragmentation. Sigh.
         *
         * This workaround assumes that the new slot starts at the same
         * address as the first existing one. If not or if some overlapping
         * slot comes around later, we will fail (not seen in practice so far)
         * - and actually require a recent KVM version. */
        if (s->broken_set_mem_region &&
689
            old.start_addr == start_addr && old.memory_size < size && add) {
690 691 692
            mem = kvm_alloc_slot(s);
            mem->memory_size = old.memory_size;
            mem->start_addr = old.start_addr;
693
            mem->ram = old.ram;
694
            mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
695 696 697 698 699 700 701 702 703

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
                        strerror(-err));
                abort();
            }

            start_addr += old.memory_size;
704
            ram += old.memory_size;
705 706 707 708 709 710 711 712 713
            size -= old.memory_size;
            continue;
        }

        /* register prefix slot */
        if (old.start_addr < start_addr) {
            mem = kvm_alloc_slot(s);
            mem->memory_size = start_addr - old.start_addr;
            mem->start_addr = old.start_addr;
714
            mem->ram = old.ram;
715
            mem->flags =  kvm_mem_flags(s, log_dirty, readonly_flag);
716 717 718 719 720

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
                        __func__, strerror(-err));
721 722 723 724 725
#ifdef TARGET_PPC
                fprintf(stderr, "%s: This is probably because your kernel's " \
                                "PAGE_SIZE is too big. Please try to use 4k " \
                                "PAGE_SIZE!\n", __func__);
#endif
726 727 728 729 730 731 732 733 734 735 736 737
                abort();
            }
        }

        /* register suffix slot */
        if (old.start_addr + old.memory_size > start_addr + size) {
            ram_addr_t size_delta;

            mem = kvm_alloc_slot(s);
            mem->start_addr = start_addr + size;
            size_delta = mem->start_addr - old.start_addr;
            mem->memory_size = old.memory_size - size_delta;
738
            mem->ram = old.ram + size_delta;
739
            mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
740 741 742 743 744 745 746 747 748 749 750

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
                        __func__, strerror(-err));
                abort();
            }
        }
    }

    /* in case the KVM bug workaround already "consumed" the new slot */
751
    if (!size) {
752
        return;
753
    }
754
    if (!add) {
755
        return;
756
    }
757 758 759
    mem = kvm_alloc_slot(s);
    mem->memory_size = size;
    mem->start_addr = start_addr;
760
    mem->ram = ram;
761
    mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
762 763 764 765 766 767 768 769 770

    err = kvm_set_user_memory_region(s, mem);
    if (err) {
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
                strerror(-err));
        abort();
    }
}

771 772 773
static void kvm_region_add(MemoryListener *listener,
                           MemoryRegionSection *section)
{
774
    memory_region_ref(section->mr);
775 776 777 778 779 780 781
    kvm_set_phys_mem(section, true);
}

static void kvm_region_del(MemoryListener *listener,
                           MemoryRegionSection *section)
{
    kvm_set_phys_mem(section, false);
782
    memory_region_unref(section->mr);
783 784 785 786
}

static void kvm_log_sync(MemoryListener *listener,
                         MemoryRegionSection *section)
787
{
788 789
    int r;

790
    r = kvm_physical_sync_dirty_bitmap(section);
791 792 793
    if (r < 0) {
        abort();
    }
794 795
}

796
static void kvm_log_global_start(struct MemoryListener *listener)
797
{
798 799 800 801
    int r;

    r = kvm_set_migration_log(1);
    assert(r >= 0);
802 803
}

804
static void kvm_log_global_stop(struct MemoryListener *listener)
805
{
806 807 808 809
    int r;

    r = kvm_set_migration_log(0);
    assert(r >= 0);
810 811
}

812 813 814 815 816 817
static void kvm_mem_ioeventfd_add(MemoryListener *listener,
                                  MemoryRegionSection *section,
                                  bool match_data, uint64_t data,
                                  EventNotifier *e)
{
    int fd = event_notifier_get_fd(e);
818 819
    int r;

820
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
821 822
                               data, true, int128_get64(section->size),
                               match_data);
823
    if (r < 0) {
824 825
        fprintf(stderr, "%s: error adding ioeventfd: %s\n",
                __func__, strerror(-r));
826 827 828 829
        abort();
    }
}

830 831 832 833
static void kvm_mem_ioeventfd_del(MemoryListener *listener,
                                  MemoryRegionSection *section,
                                  bool match_data, uint64_t data,
                                  EventNotifier *e)
834
{
835
    int fd = event_notifier_get_fd(e);
836 837
    int r;

838
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
839 840
                               data, false, int128_get64(section->size),
                               match_data);
841 842 843 844 845
    if (r < 0) {
        abort();
    }
}

846 847 848 849
static void kvm_io_ioeventfd_add(MemoryListener *listener,
                                 MemoryRegionSection *section,
                                 bool match_data, uint64_t data,
                                 EventNotifier *e)
850
{
851
    int fd = event_notifier_get_fd(e);
852 853
    int r;

854
    r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
855 856
                              data, true, int128_get64(section->size),
                              match_data);
857
    if (r < 0) {
858 859
        fprintf(stderr, "%s: error adding ioeventfd: %s\n",
                __func__, strerror(-r));
860 861 862 863
        abort();
    }
}

864 865 866 867
static void kvm_io_ioeventfd_del(MemoryListener *listener,
                                 MemoryRegionSection *section,
                                 bool match_data, uint64_t data,
                                 EventNotifier *e)
868 869

{
870
    int fd = event_notifier_get_fd(e);
871 872
    int r;

873
    r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
874 875
                              data, false, int128_get64(section->size),
                              match_data);
876 877 878 879 880
    if (r < 0) {
        abort();
    }
}

881 882 883
static MemoryListener kvm_memory_listener = {
    .region_add = kvm_region_add,
    .region_del = kvm_region_del,
884 885
    .log_start = kvm_log_start,
    .log_stop = kvm_log_stop,
886 887 888
    .log_sync = kvm_log_sync,
    .log_global_start = kvm_log_global_start,
    .log_global_stop = kvm_log_global_stop,
889 890
    .eventfd_add = kvm_mem_ioeventfd_add,
    .eventfd_del = kvm_mem_ioeventfd_del,
891 892
    .coalesced_mmio_add = kvm_coalesce_mmio_region,
    .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
893 894 895 896 897 898
    .priority = 10,
};

static MemoryListener kvm_io_listener = {
    .eventfd_add = kvm_io_ioeventfd_add,
    .eventfd_del = kvm_io_ioeventfd_del,
899
    .priority = 10,
900 901
};

902
static void kvm_handle_interrupt(CPUState *cpu, int mask)
903
{
904
    cpu->interrupt_request |= mask;
905

906
    if (!qemu_cpu_is_self(cpu)) {
907
        qemu_cpu_kick(cpu);
908 909 910
    }
}

911
int kvm_set_irq(KVMState *s, int irq, int level)
912 913 914 915
{
    struct kvm_irq_level event;
    int ret;

916
    assert(kvm_async_interrupts_enabled());
917 918 919

    event.level = level;
    event.irq = irq;
920
    ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
921
    if (ret < 0) {
922
        perror("kvm_set_irq");
923 924 925
        abort();
    }

926
    return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
927 928 929
}

#ifdef KVM_CAP_IRQ_ROUTING
930 931 932 933 934
typedef struct KVMMSIRoute {
    struct kvm_irq_routing_entry kroute;
    QTAILQ_ENTRY(KVMMSIRoute) entry;
} KVMMSIRoute;

935 936 937 938 939
static void set_gsi(KVMState *s, unsigned int gsi)
{
    s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
}

940 941 942 943 944
static void clear_gsi(KVMState *s, unsigned int gsi)
{
    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
}

945
void kvm_init_irq_routing(KVMState *s)
946
{
947
    int gsi_count, i;
948 949 950 951 952 953

    gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
    if (gsi_count > 0) {
        unsigned int gsi_bits, i;

        /* Round up so we can search ints using ffs */
954
        gsi_bits = ALIGN(gsi_count, 32);
955
        s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
956
        s->gsi_count = gsi_count;
957 958 959 960 961 962 963 964 965 966

        /* Mark any over-allocated bits as already in use */
        for (i = gsi_count; i < gsi_bits; i++) {
            set_gsi(s, i);
        }
    }

    s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
    s->nr_allocated_irq_routes = 0;

967 968 969 970
    if (!s->direct_msi) {
        for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
            QTAILQ_INIT(&s->msi_hashtab[i]);
        }
971 972
    }

973 974 975
    kvm_arch_init_irq_routing(s);
}

976
void kvm_irqchip_commit_routes(KVMState *s)
977 978 979 980 981 982 983 984
{
    int ret;

    s->irq_routes->flags = 0;
    ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
    assert(ret == 0);
}

985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
static void kvm_add_routing_entry(KVMState *s,
                                  struct kvm_irq_routing_entry *entry)
{
    struct kvm_irq_routing_entry *new;
    int n, size;

    if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
        n = s->nr_allocated_irq_routes * 2;
        if (n < 64) {
            n = 64;
        }
        size = sizeof(struct kvm_irq_routing);
        size += n * sizeof(*new);
        s->irq_routes = g_realloc(s->irq_routes, size);
        s->nr_allocated_irq_routes = n;
    }
    n = s->irq_routes->nr++;
    new = &s->irq_routes->entries[n];
1003 1004

    *new = *entry;
1005 1006 1007 1008

    set_gsi(s, entry->gsi);
}

1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
static int kvm_update_routing_entry(KVMState *s,
                                    struct kvm_irq_routing_entry *new_entry)
{
    struct kvm_irq_routing_entry *entry;
    int n;

    for (n = 0; n < s->irq_routes->nr; n++) {
        entry = &s->irq_routes->entries[n];
        if (entry->gsi != new_entry->gsi) {
            continue;
        }

1021 1022 1023 1024
        if(!memcmp(entry, new_entry, sizeof *entry)) {
            return 0;
        }

1025
        *entry = *new_entry;
1026 1027 1028 1029 1030 1031 1032 1033 1034

        kvm_irqchip_commit_routes(s);

        return 0;
    }

    return -ESRCH;
}

1035
void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
1036
{
1037
    struct kvm_irq_routing_entry e = {};
1038

1039 1040
    assert(pin < s->gsi_count);

1041 1042 1043 1044 1045 1046 1047 1048
    e.gsi = irq;
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
    e.flags = 0;
    e.u.irqchip.irqchip = irqchip;
    e.u.irqchip.pin = pin;
    kvm_add_routing_entry(s, &e);
}

1049
void kvm_irqchip_release_virq(KVMState *s, int virq)
1050 1051 1052 1053
{
    struct kvm_irq_routing_entry *e;
    int i;

1054 1055 1056 1057
    if (kvm_gsi_direct_mapping()) {
        return;
    }

1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
    for (i = 0; i < s->irq_routes->nr; i++) {
        e = &s->irq_routes->entries[i];
        if (e->gsi == virq) {
            s->irq_routes->nr--;
            *e = s->irq_routes->entries[s->irq_routes->nr];
        }
    }
    clear_gsi(s, virq);
}

static unsigned int kvm_hash_msi(uint32_t data)
{
    /* This is optimized for IA32 MSI layout. However, no other arch shall
     * repeat the mistake of not providing a direct MSI injection API. */
    return data & 0xff;
}

static void kvm_flush_dynamic_msi_routes(KVMState *s)
{
    KVMMSIRoute *route, *next;
    unsigned int hash;

    for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
        QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
            kvm_irqchip_release_virq(s, route->kroute.gsi);
            QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
            g_free(route);
        }
    }
}

static int kvm_irqchip_get_virq(KVMState *s)
{
    uint32_t *word = s->used_gsi_bitmap;
    int max_words = ALIGN(s->gsi_count, 32) / 32;
    int i, bit;
    bool retry = true;

again:
    /* Return the lowest unused GSI in the bitmap */
    for (i = 0; i < max_words; i++) {
        bit = ffs(~word[i]);
        if (!bit) {
            continue;
        }

        return bit - 1 + i * 32;
    }
1106
    if (!s->direct_msi && retry) {
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
        retry = false;
        kvm_flush_dynamic_msi_routes(s);
        goto again;
    }
    return -ENOSPC;

}

static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
{
    unsigned int hash = kvm_hash_msi(msg.data);
    KVMMSIRoute *route;

    QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
        if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
            route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1123
            route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
1124 1125 1126 1127 1128 1129 1130 1131
            return route;
        }
    }
    return NULL;
}

int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
{
1132
    struct kvm_msi msi;
1133 1134
    KVMMSIRoute *route;

1135 1136 1137
    if (s->direct_msi) {
        msi.address_lo = (uint32_t)msg.address;
        msi.address_hi = msg.address >> 32;
1138
        msi.data = le32_to_cpu(msg.data);
1139 1140 1141 1142 1143 1144
        msi.flags = 0;
        memset(msi.pad, 0, sizeof(msi.pad));

        return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
    }

1145 1146
    route = kvm_lookup_msi_route(s, msg);
    if (!route) {
1147
        int virq;
1148 1149 1150 1151 1152 1153

        virq = kvm_irqchip_get_virq(s);
        if (virq < 0) {
            return virq;
        }

1154
        route = g_malloc0(sizeof(KVMMSIRoute));
1155 1156 1157 1158 1159
        route->kroute.gsi = virq;
        route->kroute.type = KVM_IRQ_ROUTING_MSI;
        route->kroute.flags = 0;
        route->kroute.u.msi.address_lo = (uint32_t)msg.address;
        route->kroute.u.msi.address_hi = msg.address >> 32;
1160
        route->kroute.u.msi.data = le32_to_cpu(msg.data);
1161 1162

        kvm_add_routing_entry(s, &route->kroute);
1163
        kvm_irqchip_commit_routes(s);
1164 1165 1166 1167 1168 1169 1170

        QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
                           entry);
    }

    assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);

1171
    return kvm_set_irq(s, route->kroute.gsi, 1);
1172 1173
}

1174 1175
int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
{
1176
    struct kvm_irq_routing_entry kroute = {};
1177 1178
    int virq;

1179 1180 1181 1182
    if (kvm_gsi_direct_mapping()) {
        return msg.data & 0xffff;
    }

1183
    if (!kvm_gsi_routing_enabled()) {
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
        return -ENOSYS;
    }

    virq = kvm_irqchip_get_virq(s);
    if (virq < 0) {
        return virq;
    }

    kroute.gsi = virq;
    kroute.type = KVM_IRQ_ROUTING_MSI;
    kroute.flags = 0;
    kroute.u.msi.address_lo = (uint32_t)msg.address;
    kroute.u.msi.address_hi = msg.address >> 32;
1197
    kroute.u.msi.data = le32_to_cpu(msg.data);
1198 1199

    kvm_add_routing_entry(s, &kroute);
1200
    kvm_irqchip_commit_routes(s);
1201 1202 1203 1204

    return virq;
}

1205 1206
int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
{
1207
    struct kvm_irq_routing_entry kroute = {};
1208

1209 1210 1211 1212
    if (kvm_gsi_direct_mapping()) {
        return 0;
    }

1213 1214 1215 1216 1217 1218 1219 1220 1221
    if (!kvm_irqchip_in_kernel()) {
        return -ENOSYS;
    }

    kroute.gsi = virq;
    kroute.type = KVM_IRQ_ROUTING_MSI;
    kroute.flags = 0;
    kroute.u.msi.address_lo = (uint32_t)msg.address;
    kroute.u.msi.address_hi = msg.address >> 32;
1222
    kroute.u.msi.data = le32_to_cpu(msg.data);
1223 1224 1225 1226

    return kvm_update_routing_entry(s, &kroute);
}

1227 1228
static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
                                    bool assign)
1229 1230 1231 1232 1233 1234 1235
{
    struct kvm_irqfd irqfd = {
        .fd = fd,
        .gsi = virq,
        .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
    };

1236 1237 1238 1239 1240
    if (rfd != -1) {
        irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
        irqfd.resamplefd = rfd;
    }

1241
    if (!kvm_irqfds_enabled()) {
1242 1243 1244 1245 1246 1247
        return -ENOSYS;
    }

    return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
}

1248 1249
#else /* !KVM_CAP_IRQ_ROUTING */

1250
void kvm_init_irq_routing(KVMState *s)
1251 1252
{
}
1253

1254 1255 1256 1257
void kvm_irqchip_release_virq(KVMState *s, int virq)
{
}

1258 1259 1260 1261
int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
{
    abort();
}
1262 1263 1264

int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
{
1265
    return -ENOSYS;
1266
}
1267 1268 1269 1270 1271

static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
{
    abort();
}
1272 1273 1274 1275 1276

int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
{
    return -ENOSYS;
}
1277 1278
#endif /* !KVM_CAP_IRQ_ROUTING */

1279 1280
int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
                                   EventNotifier *rn, int virq)
1281
{
1282 1283
    return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
           rn ? event_notifier_get_fd(rn) : -1, virq, true);
1284 1285
}

Jan Kiszka's avatar
Jan Kiszka committed
1286
int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
1287
{
1288 1289
    return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
           false);
1290 1291
}

1292 1293 1294 1295
static int kvm_irqchip_create(KVMState *s)
{
    int ret;

1296
    if (!qemu_opt_get_bool(qemu_get_machine_opts(), "kernel_irqchip", true) ||
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
        !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
        return 0;
    }

    ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
    if (ret < 0) {
        fprintf(stderr, "Create kernel irqchip failed\n");
        return ret;
    }

1307
    kvm_kernel_irqchip = true;
1308 1309 1310 1311
    /* If we have an in-kernel IRQ chip then we must have asynchronous
     * interrupt delivery (though the reverse is not necessarily true)
     */
    kvm_async_interrupts_allowed = true;
1312
    kvm_halt_in_kernel_allowed = true;
1313 1314 1315 1316 1317 1318

    kvm_init_irq_routing(s);

    return 0;
}

1319 1320 1321 1322 1323
/* Find number of supported CPUs using the recommended
 * procedure from the kernel API documentation to cope with
 * older kernels that may be missing capabilities.
 */
static int kvm_recommended_vcpus(KVMState *s)
1324
{
1325 1326 1327
    int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
    return (ret) ? ret : 4;
}
1328

1329 1330 1331 1332
static int kvm_max_vcpus(KVMState *s)
{
    int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
    return (ret) ? ret : kvm_recommended_vcpus(s);
1333 1334
}

1335
int kvm_init(void)
1336
{
1337 1338 1339
    static const char upgrade_note[] =
        "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
        "(see http://sourceforge.net/projects/kvm).\n";
1340 1341 1342 1343 1344 1345 1346 1347 1348
    struct {
        const char *name;
        int num;
    } num_cpus[] = {
        { "SMP",          smp_cpus },
        { "hotpluggable", max_cpus },
        { NULL, }
    }, *nc = num_cpus;
    int soft_vcpus_limit, hard_vcpus_limit;
1349
    KVMState *s;