kvm-all.c 51.2 KB
Newer Older
aliguori's avatar
aliguori committed
1
2
3
4
/*
 * QEMU KVM support
 *
 * Copyright IBM, Corp. 2008
5
 *           Red Hat, Inc. 2008
aliguori's avatar
aliguori committed
6
7
8
 *
 * Authors:
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Glauber Costa     <gcosta@redhat.com>
aliguori's avatar
aliguori committed
10
11
12
13
14
15
16
17
18
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */

#include <sys/types.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
19
#include <stdarg.h>
aliguori's avatar
aliguori committed
20
21
22
23

#include <linux/kvm.h>

#include "qemu-common.h"
24
25
26
#include "qemu/atomic.h"
#include "qemu/option.h"
#include "qemu/config-file.h"
27
#include "sysemu/sysemu.h"
Jan Kiszka's avatar
Jan Kiszka committed
28
#include "hw/hw.h"
29
#include "hw/pci/msi.h"
30
#include "exec/gdbstub.h"
31
#include "sysemu/kvm.h"
32
#include "qemu/bswap.h"
33
34
#include "exec/memory.h"
#include "exec/address-spaces.h"
35
#include "qemu/event_notifier.h"
aliguori's avatar
aliguori committed
36

37
38
39
40
41
/* This check must be after config-host.h is included */
#ifdef CONFIG_EVENTFD
#include <sys/eventfd.h>
#endif

42
43
44
45
#ifdef CONFIG_VALGRIND_H
#include <valgrind/memcheck.h>
#endif

46
/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
aliguori's avatar
aliguori committed
47
48
#define PAGE_SIZE TARGET_PAGE_SIZE

aliguori's avatar
aliguori committed
49
50
51
//#define DEBUG_KVM

#ifdef DEBUG_KVM
52
#define DPRINTF(fmt, ...) \
aliguori's avatar
aliguori committed
53
54
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
#else
55
#define DPRINTF(fmt, ...) \
aliguori's avatar
aliguori committed
56
57
58
    do { } while (0)
#endif

59
60
#define KVM_MSI_HASHTAB_SIZE    256

aliguori's avatar
aliguori committed
61
62
typedef struct KVMSlot
{
63
    hwaddr start_addr;
64
    ram_addr_t memory_size;
65
    void *ram;
aliguori's avatar
aliguori committed
66
67
68
    int slot;
    int flags;
} KVMSlot;
aliguori's avatar
aliguori committed
69

70
71
typedef struct kvm_dirty_log KVMDirtyLog;

aliguori's avatar
aliguori committed
72
73
74
75
76
struct KVMState
{
    KVMSlot slots[32];
    int fd;
    int vmfd;
aliguori's avatar
aliguori committed
77
    int coalesced_mmio;
78
    struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
79
    bool coalesced_flush_in_progress;
80
    int broken_set_mem_region;
81
    int migration_log;
82
    int vcpu_events;
83
    int robust_singlestep;
84
    int debugregs;
85
86
87
#ifdef KVM_CAP_SET_GUEST_DEBUG
    struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
#endif
88
    int pit_state2;
89
    int xsave, xcrs;
90
    int many_ioeventfds;
91
    int intx_set_mask;
92
93
94
    /* The man page (and posix) say ioctl numbers are signed int, but
     * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
     * unsigned, and treating them as signed here can break things */
95
    unsigned irq_set_ioctl;
96
97
98
99
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing *irq_routes;
    int nr_allocated_irq_routes;
    uint32_t *used_gsi_bitmap;
100
    unsigned int gsi_count;
101
    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
102
    bool direct_msi;
103
#endif
aliguori's avatar
aliguori committed
104
105
};

106
KVMState *kvm_state;
107
bool kvm_kernel_irqchip;
108
bool kvm_async_interrupts_allowed;
109
bool kvm_irqfds_allowed;
110
bool kvm_msi_via_irqfd_allowed;
111
bool kvm_gsi_routing_allowed;
aliguori's avatar
aliguori committed
112

113
114
115
116
117
118
static const KVMCapabilityInfo kvm_required_capabilites[] = {
    KVM_CAP_INFO(USER_MEMORY),
    KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
    KVM_CAP_LAST_INFO
};

aliguori's avatar
aliguori committed
119
120
121
122
123
static KVMSlot *kvm_alloc_slot(KVMState *s)
{
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
Jan Kiszka's avatar
Jan Kiszka committed
124
        if (s->slots[i].memory_size == 0) {
aliguori's avatar
aliguori committed
125
            return &s->slots[i];
Jan Kiszka's avatar
Jan Kiszka committed
126
        }
aliguori's avatar
aliguori committed
127
128
    }

129
130
131
132
133
    fprintf(stderr, "%s: no free slot available\n", __func__);
    abort();
}

static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
134
135
                                         hwaddr start_addr,
                                         hwaddr end_addr)
136
137
138
139
140
141
142
143
144
145
146
147
{
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        KVMSlot *mem = &s->slots[i];

        if (start_addr == mem->start_addr &&
            end_addr == mem->start_addr + mem->memory_size) {
            return mem;
        }
    }

aliguori's avatar
aliguori committed
148
149
150
    return NULL;
}

151
152
153
154
/*
 * Find overlapping slot with lowest start address
 */
static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
155
156
                                            hwaddr start_addr,
                                            hwaddr end_addr)
aliguori's avatar
aliguori committed
157
{
158
    KVMSlot *found = NULL;
aliguori's avatar
aliguori committed
159
160
161
162
163
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        KVMSlot *mem = &s->slots[i];

164
165
166
167
168
169
170
171
172
        if (mem->memory_size == 0 ||
            (found && found->start_addr < mem->start_addr)) {
            continue;
        }

        if (end_addr > mem->start_addr &&
            start_addr < mem->start_addr + mem->memory_size) {
            found = mem;
        }
aliguori's avatar
aliguori committed
173
174
    }

175
    return found;
aliguori's avatar
aliguori committed
176
177
}

178
int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
179
                                       hwaddr *phys_addr)
180
181
182
183
184
185
{
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        KVMSlot *mem = &s->slots[i];

186
187
        if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
            *phys_addr = mem->start_addr + (ram - mem->ram);
188
189
190
191
192
193
194
            return 1;
        }
    }

    return 0;
}

195
196
197
198
199
200
201
static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
{
    struct kvm_userspace_memory_region mem;

    mem.slot = slot->slot;
    mem.guest_phys_addr = slot->start_addr;
    mem.memory_size = slot->memory_size;
202
    mem.userspace_addr = (unsigned long)slot->ram;
203
    mem.flags = slot->flags;
204
205
206
    if (s->migration_log) {
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
    }
207
208
209
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
}

Jan Kiszka's avatar
Jan Kiszka committed
210
211
static void kvm_reset_vcpu(void *opaque)
{
212
    CPUState *cpu = opaque;
Jan Kiszka's avatar
Jan Kiszka committed
213

214
    kvm_arch_reset_vcpu(cpu);
Jan Kiszka's avatar
Jan Kiszka committed
215
}
216

217
int kvm_init_vcpu(CPUState *cpu)
aliguori's avatar
aliguori committed
218
219
220
221
222
{
    KVMState *s = kvm_state;
    long mmap_size;
    int ret;

223
    DPRINTF("kvm_init_vcpu\n");
aliguori's avatar
aliguori committed
224

225
    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, cpu->cpu_index);
aliguori's avatar
aliguori committed
226
    if (ret < 0) {
227
        DPRINTF("kvm_create_vcpu failed\n");
aliguori's avatar
aliguori committed
228
229
230
        goto err;
    }

231
    cpu->kvm_fd = ret;
232
    cpu->kvm_state = s;
233
    cpu->kvm_vcpu_dirty = true;
aliguori's avatar
aliguori committed
234
235
236

    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
    if (mmap_size < 0) {
237
        ret = mmap_size;
238
        DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
aliguori's avatar
aliguori committed
239
240
241
        goto err;
    }

242
    cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
243
                        cpu->kvm_fd, 0);
244
    if (cpu->kvm_run == MAP_FAILED) {
aliguori's avatar
aliguori committed
245
        ret = -errno;
246
        DPRINTF("mmap'ing vcpu state failed\n");
aliguori's avatar
aliguori committed
247
248
249
        goto err;
    }

Jan Kiszka's avatar
Jan Kiszka committed
250
251
    if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
        s->coalesced_mmio_ring =
252
            (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
Jan Kiszka's avatar
Jan Kiszka committed
253
    }
254

255
    ret = kvm_arch_init_vcpu(cpu);
Jan Kiszka's avatar
Jan Kiszka committed
256
    if (ret == 0) {
257
258
        qemu_register_reset(kvm_reset_vcpu, cpu);
        kvm_arch_reset_vcpu(cpu);
Jan Kiszka's avatar
Jan Kiszka committed
259
    }
aliguori's avatar
aliguori committed
260
261
262
263
err:
    return ret;
}

264
265
266
/*
 * dirty pages logging control
 */
267
268
269
270
271
272
273

static int kvm_mem_flags(KVMState *s, bool log_dirty)
{
    return log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
}

static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
274
275
{
    KVMState *s = kvm_state;
276
    int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
277
278
279
    int old_flags;

    old_flags = mem->flags;
280

281
    flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty);
282
283
    mem->flags = flags;

284
285
286
287
    /* If nothing changed effectively, no need to issue ioctl */
    if (s->migration_log) {
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
    }
288

289
    if (flags == old_flags) {
290
        return 0;
291
292
    }

293
294
295
    return kvm_set_user_memory_region(s, mem);
}

296
static int kvm_dirty_pages_log_change(hwaddr phys_addr,
297
298
299
300
301
302
303
304
                                      ram_addr_t size, bool log_dirty)
{
    KVMState *s = kvm_state;
    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);

    if (mem == NULL)  {
        fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
                TARGET_FMT_plx "\n", __func__, phys_addr,
305
                (hwaddr)(phys_addr + size - 1));
306
307
308
309
310
        return -EINVAL;
    }
    return kvm_slot_dirty_pages_log_change(mem, log_dirty);
}

311
312
static void kvm_log_start(MemoryListener *listener,
                          MemoryRegionSection *section)
313
{
314
315
316
317
318
319
320
    int r;

    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
                                   section->size, true);
    if (r < 0) {
        abort();
    }
321
322
}

323
324
static void kvm_log_stop(MemoryListener *listener,
                          MemoryRegionSection *section)
325
{
326
327
328
329
330
331
332
    int r;

    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
                                   section->size, false);
    if (r < 0) {
        abort();
    }
333
334
}

335
static int kvm_set_migration_log(int enable)
336
337
338
339
340
341
342
343
344
345
{
    KVMState *s = kvm_state;
    KVMSlot *mem;
    int i, err;

    s->migration_log = enable;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        mem = &s->slots[i];

346
347
348
        if (!mem->memory_size) {
            continue;
        }
349
350
351
352
353
354
355
356
357
358
359
        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
            continue;
        }
        err = kvm_set_user_memory_region(s, mem);
        if (err) {
            return err;
        }
    }
    return 0;
}

360
/* get kvm's dirty pages bitmap and update qemu's */
361
362
static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
                                         unsigned long *bitmap)
363
{
364
    unsigned int i, j;
365
    unsigned long page_number, c;
366
    hwaddr addr, addr1;
367
    unsigned int len = ((section->size / getpagesize()) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
368
    unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
369
370
371
372
373
374
375
376
377
378
379

    /*
     * bitmap-traveling is faster than memory-traveling (for addr...)
     * especially when most of the memory is not dirty.
     */
    for (i = 0; i < len; i++) {
        if (bitmap[i] != 0) {
            c = leul_to_cpu(bitmap[i]);
            do {
                j = ffsl(c) - 1;
                c &= ~(1ul << j);
380
                page_number = (i * HOST_LONG_BITS + j) * hpratio;
381
                addr1 = page_number * TARGET_PAGE_SIZE;
382
                addr = section->offset_within_region + addr1;
383
384
                memory_region_set_dirty(section->mr, addr,
                                        TARGET_PAGE_SIZE * hpratio);
385
386
387
388
            } while (c != 0);
        }
    }
    return 0;
389
390
}

391
392
#define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))

393
394
/**
 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
395
396
397
 * This function updates qemu's dirty bitmap using
 * memory_region_set_dirty().  This means all bits are set
 * to dirty.
398
 *
399
 * @start_add: start of logged region.
400
401
 * @end_addr: end of logged region.
 */
402
static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
403
404
{
    KVMState *s = kvm_state;
405
406
407
408
    unsigned long size, allocated_size = 0;
    KVMDirtyLog d;
    KVMSlot *mem;
    int ret = 0;
409
410
    hwaddr start_addr = section->offset_within_address_space;
    hwaddr end_addr = start_addr + section->size;
411

412
413
414
415
416
417
    d.dirty_bitmap = NULL;
    while (start_addr < end_addr) {
        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
        if (mem == NULL) {
            break;
        }
418

419
420
421
422
423
424
425
426
427
428
429
430
431
432
        /* XXX bad kernel interface alert
         * For dirty bitmap, kernel allocates array of size aligned to
         * bits-per-long.  But for case when the kernel is 64bits and
         * the userspace is 32bits, userspace can't align to the same
         * bits-per-long, since sizeof(long) is different between kernel
         * and user space.  This way, userspace will provide buffer which
         * may be 4 bytes less than the kernel will use, resulting in
         * userspace memory corruption (which is not detectable by valgrind
         * too, in most cases).
         * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
         * a hope that sizeof(long) wont become >8 any time soon.
         */
        size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
                     /*HOST_LONG_BITS*/ 64) / 8;
433
        if (!d.dirty_bitmap) {
434
            d.dirty_bitmap = g_malloc(size);
435
        } else if (size > allocated_size) {
436
            d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
437
438
439
        }
        allocated_size = size;
        memset(d.dirty_bitmap, 0, allocated_size);
440

441
        d.slot = mem->slot;
442

443
        if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
444
            DPRINTF("ioctl failed %d\n", errno);
445
446
447
            ret = -1;
            break;
        }
448

449
        kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
450
        start_addr = mem->start_addr + mem->memory_size;
451
    }
452
    g_free(d.dirty_bitmap);
453
454

    return ret;
455
456
}

457
458
static void kvm_coalesce_mmio_region(MemoryListener *listener,
                                     MemoryRegionSection *secion,
459
                                     hwaddr start, hwaddr size)
aliguori's avatar
aliguori committed
460
461
462
463
464
465
466
467
{
    KVMState *s = kvm_state;

    if (s->coalesced_mmio) {
        struct kvm_coalesced_mmio_zone zone;

        zone.addr = start;
        zone.size = size;
468
        zone.pad = 0;
aliguori's avatar
aliguori committed
469

470
        (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
aliguori's avatar
aliguori committed
471
472
473
    }
}

474
475
static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
                                       MemoryRegionSection *secion,
476
                                       hwaddr start, hwaddr size)
aliguori's avatar
aliguori committed
477
478
479
480
481
482
483
484
{
    KVMState *s = kvm_state;

    if (s->coalesced_mmio) {
        struct kvm_coalesced_mmio_zone zone;

        zone.addr = start;
        zone.size = size;
485
        zone.pad = 0;
aliguori's avatar
aliguori committed
486

487
        (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
aliguori's avatar
aliguori committed
488
489
490
    }
}

491
492
493
494
495
496
497
498
499
500
501
502
int kvm_check_extension(KVMState *s, unsigned int extension)
{
    int ret;

    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
    if (ret < 0) {
        ret = 0;
    }

    return ret;
}

503
504
static int kvm_check_many_ioeventfds(void)
{
505
506
507
508
509
    /* Userspace can use ioeventfd for io notification.  This requires a host
     * that supports eventfd(2) and an I/O thread; since eventfd does not
     * support SIGIO it cannot interrupt the vcpu.
     *
     * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
510
511
     * can avoid creating too many ioeventfds.
     */
512
#if defined(CONFIG_EVENTFD)
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
    int ioeventfds[7];
    int i, ret = 0;
    for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
        ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
        if (ioeventfds[i] < 0) {
            break;
        }
        ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
        if (ret < 0) {
            close(ioeventfds[i]);
            break;
        }
    }

    /* Decide whether many devices are supported or not */
    ret = i == ARRAY_SIZE(ioeventfds);

    while (i-- > 0) {
        kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
        close(ioeventfds[i]);
    }
    return ret;
#else
    return 0;
#endif
}

540
541
542
543
544
545
546
547
548
549
550
551
static const KVMCapabilityInfo *
kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
{
    while (list->name) {
        if (!kvm_check_extension(s, list->value)) {
            return list;
        }
        list++;
    }
    return NULL;
}

552
static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
553
554
555
556
{
    KVMState *s = kvm_state;
    KVMSlot *mem, old;
    int err;
557
558
    MemoryRegion *mr = section->mr;
    bool log_dirty = memory_region_is_logging(mr);
559
    hwaddr start_addr = section->offset_within_address_space;
560
    ram_addr_t size = section->size;
561
    void *ram = NULL;
Avi Kivity's avatar
Avi Kivity committed
562
    unsigned delta;
563

564
565
    /* kvm works in page size chunks, but the function may be called
       with sub-page size and unaligned start address. */
Avi Kivity's avatar
Avi Kivity committed
566
567
568
569
570
571
572
573
574
575
    delta = TARGET_PAGE_ALIGN(size) - size;
    if (delta > size) {
        return;
    }
    start_addr += delta;
    size -= delta;
    size &= TARGET_PAGE_MASK;
    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
        return;
    }
576

577
578
    if (!memory_region_is_ram(mr)) {
        return;
579
580
    }

Avi Kivity's avatar
Avi Kivity committed
581
    ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
582

583
584
585
586
587
588
    while (1) {
        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
        if (!mem) {
            break;
        }

589
        if (add && start_addr >= mem->start_addr &&
590
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
591
            (ram - start_addr == mem->ram - mem->start_addr)) {
592
            /* The new slot fits into the existing one and comes with
593
594
             * identical parameters - update flags and done. */
            kvm_slot_dirty_pages_log_change(mem, log_dirty);
595
596
597
598
599
            return;
        }

        old = *mem;

600
601
602
603
        if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
            kvm_physical_sync_dirty_bitmap(section);
        }

604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
        /* unregister the overlapping slot */
        mem->memory_size = 0;
        err = kvm_set_user_memory_region(s, mem);
        if (err) {
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
                    __func__, strerror(-err));
            abort();
        }

        /* Workaround for older KVM versions: we can't join slots, even not by
         * unregistering the previous ones and then registering the larger
         * slot. We have to maintain the existing fragmentation. Sigh.
         *
         * This workaround assumes that the new slot starts at the same
         * address as the first existing one. If not or if some overlapping
         * slot comes around later, we will fail (not seen in practice so far)
         * - and actually require a recent KVM version. */
        if (s->broken_set_mem_region &&
622
            old.start_addr == start_addr && old.memory_size < size && add) {
623
624
625
            mem = kvm_alloc_slot(s);
            mem->memory_size = old.memory_size;
            mem->start_addr = old.start_addr;
626
            mem->ram = old.ram;
627
            mem->flags = kvm_mem_flags(s, log_dirty);
628
629
630
631
632
633
634
635
636

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
                        strerror(-err));
                abort();
            }

            start_addr += old.memory_size;
637
            ram += old.memory_size;
638
639
640
641
642
643
644
645
646
            size -= old.memory_size;
            continue;
        }

        /* register prefix slot */
        if (old.start_addr < start_addr) {
            mem = kvm_alloc_slot(s);
            mem->memory_size = start_addr - old.start_addr;
            mem->start_addr = old.start_addr;
647
            mem->ram = old.ram;
648
            mem->flags =  kvm_mem_flags(s, log_dirty);
649
650
651
652
653

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
                        __func__, strerror(-err));
654
655
656
657
658
#ifdef TARGET_PPC
                fprintf(stderr, "%s: This is probably because your kernel's " \
                                "PAGE_SIZE is too big. Please try to use 4k " \
                                "PAGE_SIZE!\n", __func__);
#endif
659
660
661
662
663
664
665
666
667
668
669
670
                abort();
            }
        }

        /* register suffix slot */
        if (old.start_addr + old.memory_size > start_addr + size) {
            ram_addr_t size_delta;

            mem = kvm_alloc_slot(s);
            mem->start_addr = start_addr + size;
            size_delta = mem->start_addr - old.start_addr;
            mem->memory_size = old.memory_size - size_delta;
671
            mem->ram = old.ram + size_delta;
672
            mem->flags = kvm_mem_flags(s, log_dirty);
673
674
675
676
677
678
679
680
681
682
683

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
                        __func__, strerror(-err));
                abort();
            }
        }
    }

    /* in case the KVM bug workaround already "consumed" the new slot */
Jan Kiszka's avatar
Jan Kiszka committed
684
    if (!size) {
685
        return;
Jan Kiszka's avatar
Jan Kiszka committed
686
    }
687
    if (!add) {
688
        return;
Jan Kiszka's avatar
Jan Kiszka committed
689
    }
690
691
692
    mem = kvm_alloc_slot(s);
    mem->memory_size = size;
    mem->start_addr = start_addr;
693
    mem->ram = ram;
694
    mem->flags = kvm_mem_flags(s, log_dirty);
695
696
697
698
699
700
701
702
703

    err = kvm_set_user_memory_region(s, mem);
    if (err) {
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
                strerror(-err));
        abort();
    }
}

704
705
706
707
708
709
710
711
712
713
714
715
716
717
static void kvm_region_add(MemoryListener *listener,
                           MemoryRegionSection *section)
{
    kvm_set_phys_mem(section, true);
}

static void kvm_region_del(MemoryListener *listener,
                           MemoryRegionSection *section)
{
    kvm_set_phys_mem(section, false);
}

static void kvm_log_sync(MemoryListener *listener,
                         MemoryRegionSection *section)
718
{
719
720
    int r;

721
    r = kvm_physical_sync_dirty_bitmap(section);
722
723
724
    if (r < 0) {
        abort();
    }
725
726
}

727
static void kvm_log_global_start(struct MemoryListener *listener)
728
{
729
730
731
732
    int r;

    r = kvm_set_migration_log(1);
    assert(r >= 0);
733
734
}

735
static void kvm_log_global_stop(struct MemoryListener *listener)
736
{
737
738
739
740
    int r;

    r = kvm_set_migration_log(0);
    assert(r >= 0);
741
742
}

743
744
745
746
747
748
static void kvm_mem_ioeventfd_add(MemoryListener *listener,
                                  MemoryRegionSection *section,
                                  bool match_data, uint64_t data,
                                  EventNotifier *e)
{
    int fd = event_notifier_get_fd(e);
749
750
    int r;

751
    assert(match_data && section->size <= 8);
752

753
754
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
                               data, true, section->size);
755
756
757
758
759
    if (r < 0) {
        abort();
    }
}

760
761
762
763
static void kvm_mem_ioeventfd_del(MemoryListener *listener,
                                  MemoryRegionSection *section,
                                  bool match_data, uint64_t data,
                                  EventNotifier *e)
764
{
765
    int fd = event_notifier_get_fd(e);
766
767
    int r;

768
769
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
                               data, false, section->size);
770
771
772
773
774
    if (r < 0) {
        abort();
    }
}

775
776
777
778
static void kvm_io_ioeventfd_add(MemoryListener *listener,
                                 MemoryRegionSection *section,
                                 bool match_data, uint64_t data,
                                 EventNotifier *e)
779
{
780
    int fd = event_notifier_get_fd(e);
781
782
783
784
785
786
787
788
789
790
791
    int r;

    assert(match_data && section->size == 2);

    r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
                                   data, true);
    if (r < 0) {
        abort();
    }
}

792
793
794
795
static void kvm_io_ioeventfd_del(MemoryListener *listener,
                                 MemoryRegionSection *section,
                                 bool match_data, uint64_t data,
                                 EventNotifier *e)
796
797

{
798
    int fd = event_notifier_get_fd(e);
799
800
801
802
803
804
805
806
807
    int r;

    r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
                                   data, false);
    if (r < 0) {
        abort();
    }
}

808
809
810
static MemoryListener kvm_memory_listener = {
    .region_add = kvm_region_add,
    .region_del = kvm_region_del,
811
812
    .log_start = kvm_log_start,
    .log_stop = kvm_log_stop,
813
814
815
    .log_sync = kvm_log_sync,
    .log_global_start = kvm_log_global_start,
    .log_global_stop = kvm_log_global_stop,
816
817
    .eventfd_add = kvm_mem_ioeventfd_add,
    .eventfd_del = kvm_mem_ioeventfd_del,
818
819
    .coalesced_mmio_add = kvm_coalesce_mmio_region,
    .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
820
821
822
823
824
825
    .priority = 10,
};

static MemoryListener kvm_io_listener = {
    .eventfd_add = kvm_io_ioeventfd_add,
    .eventfd_del = kvm_io_ioeventfd_del,
826
    .priority = 10,
827
828
};

829
static void kvm_handle_interrupt(CPUArchState *env, int mask)
830
{
831
832
    CPUState *cpu = ENV_GET_CPU(env);

833
834
    env->interrupt_request |= mask;

835
    if (!qemu_cpu_is_self(cpu)) {
836
        qemu_cpu_kick(cpu);
837
838
839
    }
}

840
int kvm_set_irq(KVMState *s, int irq, int level)
841
842
843
844
{
    struct kvm_irq_level event;
    int ret;

845
    assert(kvm_async_interrupts_enabled());
846
847
848

    event.level = level;
    event.irq = irq;
849
    ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
850
    if (ret < 0) {
851
        perror("kvm_set_irq");
852
853
854
        abort();
    }

855
    return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
856
857
858
}

#ifdef KVM_CAP_IRQ_ROUTING
859
860
861
862
863
typedef struct KVMMSIRoute {
    struct kvm_irq_routing_entry kroute;
    QTAILQ_ENTRY(KVMMSIRoute) entry;
} KVMMSIRoute;

864
865
866
867
868
static void set_gsi(KVMState *s, unsigned int gsi)
{
    s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
}

869
870
871
872
873
static void clear_gsi(KVMState *s, unsigned int gsi)
{
    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
}

874
875
static void kvm_init_irq_routing(KVMState *s)
{
876
    int gsi_count, i;
877
878
879
880
881
882

    gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
    if (gsi_count > 0) {
        unsigned int gsi_bits, i;

        /* Round up so we can search ints using ffs */
883
        gsi_bits = ALIGN(gsi_count, 32);
884
        s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
885
        s->gsi_count = gsi_count;
886
887
888
889
890
891
892
893
894
895

        /* Mark any over-allocated bits as already in use */
        for (i = gsi_count; i < gsi_bits; i++) {
            set_gsi(s, i);
        }
    }

    s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
    s->nr_allocated_irq_routes = 0;

896
897
898
899
    if (!s->direct_msi) {
        for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
            QTAILQ_INIT(&s->msi_hashtab[i]);
        }
900
901
    }

902
903
904
    kvm_arch_init_irq_routing(s);
}

905
906
907
908
909
910
911
912
913
static void kvm_irqchip_commit_routes(KVMState *s)
{
    int ret;

    s->irq_routes->flags = 0;
    ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
    assert(ret == 0);
}

914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
static void kvm_add_routing_entry(KVMState *s,
                                  struct kvm_irq_routing_entry *entry)
{
    struct kvm_irq_routing_entry *new;
    int n, size;

    if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
        n = s->nr_allocated_irq_routes * 2;
        if (n < 64) {
            n = 64;
        }
        size = sizeof(struct kvm_irq_routing);
        size += n * sizeof(*new);
        s->irq_routes = g_realloc(s->irq_routes, size);
        s->nr_allocated_irq_routes = n;
    }
    n = s->irq_routes->nr++;
    new = &s->irq_routes->entries[n];
    memset(new, 0, sizeof(*new));
    new->gsi = entry->gsi;
    new->type = entry->type;
    new->flags = entry->flags;
    new->u = entry->u;

    set_gsi(s, entry->gsi);
939
940

    kvm_irqchip_commit_routes(s);
941
942
}

943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
static int kvm_update_routing_entry(KVMState *s,
                                    struct kvm_irq_routing_entry *new_entry)
{
    struct kvm_irq_routing_entry *entry;
    int n;

    for (n = 0; n < s->irq_routes->nr; n++) {
        entry = &s->irq_routes->entries[n];
        if (entry->gsi != new_entry->gsi) {
            continue;
        }

        entry->type = new_entry->type;
        entry->flags = new_entry->flags;
        entry->u = new_entry->u;

        kvm_irqchip_commit_routes(s);

        return 0;
    }

    return -ESRCH;
}

967
void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
968
969
970
{
    struct kvm_irq_routing_entry e;

971
972
    assert(pin < s->gsi_count);

973
974
975
976
977
978
979
980
    e.gsi = irq;
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
    e.flags = 0;
    e.u.irqchip.irqchip = irqchip;
    e.u.irqchip.pin = pin;
    kvm_add_routing_entry(s, &e);
}

981
void kvm_irqchip_release_virq(KVMState *s, int virq)
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
{
    struct kvm_irq_routing_entry *e;
    int i;

    for (i = 0; i < s->irq_routes->nr; i++) {
        e = &s->irq_routes->entries[i];
        if (e->gsi == virq) {
            s->irq_routes->nr--;
            *e = s->irq_routes->entries[s->irq_routes->nr];
        }
    }
    clear_gsi(s, virq);
}

static unsigned int kvm_hash_msi(uint32_t data)
{
    /* This is optimized for IA32 MSI layout. However, no other arch shall
     * repeat the mistake of not providing a direct MSI injection API. */
    return data & 0xff;