@@ -2297,8 +2297,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn_start, kvm_pfn_t pf
kunmap_local(vaddr);
}
- ret = rmp_make_private(pfn + i, gfn << PAGE_SHIFT, PG_LEVEL_4K,
- sev_get_asid(kvm), true);
+ ret = guest_memfd_make_inaccessible(pfn_folio(pfn));
if (ret)
goto err;
@@ -106,6 +106,7 @@ config KVM_GENERIC_MEMORY_ATTRIBUTES
config KVM_PRIVATE_MEM
select XARRAY_MULTI
+ select GUEST_MEMFD
bool
config KVM_GENERIC_PRIVATE_MEM
@@ -1,9 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
-#include <linux/backing-dev.h>
-#include <linux/falloc.h>
+#include <linux/guest_memfd.h>
#include <linux/kvm_host.h>
#include <linux/pagemap.h>
-#include <linux/anon_inodes.h>
#include "kvm_mm.h"
@@ -13,6 +11,13 @@ struct kvm_gmem {
struct list_head entry;
};
+static inline struct kvm_gmem *inode_to_kvm_gmem(struct inode *inode)
+{
+ struct list_head *gmem_list = &inode->i_mapping->i_private_list;
+
+ return list_first_entry_or_null(gmem_list, struct kvm_gmem, entry);
+}
+
/**
* folio_file_pfn - like folio_file_page, but return a pfn.
* @folio: The folio which contains this index.
@@ -25,12 +30,11 @@ static inline kvm_pfn_t folio_file_pfn(struct folio *folio, pgoff_t index)
return folio_pfn(folio) + (index & (folio_nr_pages(folio) - 1));
}
-static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
- pgoff_t index, struct folio *folio)
+static int kvm_gmem_prepare_inaccessible(struct inode *inode, struct folio *folio)
{
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE
- kvm_pfn_t pfn = folio_file_pfn(folio, index);
- gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff;
+ kvm_pfn_t pfn = folio_file_pfn(folio, 0);
+ gfn_t gfn = slot->base_gfn + folio_index(folio) - slot->gmem.pgoff;
int rc = kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio));
if (rc) {
pr_warn_ratelimited("gmem: Failed to prepare folio for index %lx GFN %llx PFN %llx error %d.\n",
@@ -42,67 +46,7 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo
return 0;
}
-static inline void kvm_gmem_mark_prepared(struct folio *folio)
-{
- folio_mark_uptodate(folio);
-}
-
-/*
- * Process @folio, which contains @gfn, so that the guest can use it.
- * The folio must be locked and the gfn must be contained in @slot.
- * On successful return the guest sees a zero page so as to avoid
- * leaking host data and the up-to-date flag is set.
- */
-static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
- gfn_t gfn, struct folio *folio)
-{
- unsigned long nr_pages, i;
- pgoff_t index;
- int r;
-
- nr_pages = folio_nr_pages(folio);
- for (i = 0; i < nr_pages; i++)
- clear_highpage(folio_page(folio, i));
-
- /*
- * Preparing huge folios should always be safe, since it should
- * be possible to split them later if needed.
- *
- * Right now the folio order is always going to be zero, but the
- * code is ready for huge folios. The only assumption is that
- * the base pgoff of memslots is naturally aligned with the
- * requested page order, ensuring that huge folios can also use
- * huge page table entries for GPA->HPA mapping.
- *
- * The order will be passed when creating the guest_memfd, and
- * checked when creating memslots.
- */
- WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, 1 << folio_order(folio)));
- index = gfn - slot->base_gfn + slot->gmem.pgoff;
- index = ALIGN_DOWN(index, 1 << folio_order(folio));
- r = __kvm_gmem_prepare_folio(kvm, slot, index, folio);
- if (!r)
- kvm_gmem_mark_prepared(folio);
-
- return r;
-}
-
-/*
- * Returns a locked folio on success. The caller is responsible for
- * setting the up-to-date flag before the memory is mapped into the guest.
- * There is no backing storage for the memory, so the folio will remain
- * up-to-date until it's removed.
- *
- * Ignore accessed, referenced, and dirty flags. The memory is
- * unevictable and there is no storage to write back to.
- */
-static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)
-{
- /* TODO: Support huge pages. */
- return filemap_grab_folio(inode->i_mapping, index);
-}
-
-static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,
+static void __kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,
pgoff_t end)
{
bool flush = false, found_memslot = false;
@@ -137,118 +81,38 @@ static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,
KVM_MMU_UNLOCK(kvm);
}
-static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start,
- pgoff_t end)
+static int kvm_gmem_invalidate_begin(struct inode *inode, pgoff_t offset, unsigned long nr)
{
- struct kvm *kvm = gmem->kvm;
-
- if (xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {
- KVM_MMU_LOCK(kvm);
- kvm_mmu_invalidate_end(kvm);
- KVM_MMU_UNLOCK(kvm);
- }
-}
-
-static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
-{
- struct list_head *gmem_list = &inode->i_mapping->i_private_list;
- pgoff_t start = offset >> PAGE_SHIFT;
- pgoff_t end = (offset + len) >> PAGE_SHIFT;
struct kvm_gmem *gmem;
- /*
- * Bindings must be stable across invalidation to ensure the start+end
- * are balanced.
- */
- filemap_invalidate_lock(inode->i_mapping);
-
- list_for_each_entry(gmem, gmem_list, entry)
- kvm_gmem_invalidate_begin(gmem, start, end);
-
- truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1);
-
- list_for_each_entry(gmem, gmem_list, entry)
- kvm_gmem_invalidate_end(gmem, start, end);
-
- filemap_invalidate_unlock(inode->i_mapping);
+ list_for_each_entry(gmem, &inode->i_mapping->i_private_list, entry)
+ __kvm_gmem_invalidate_begin(gmem, offset, offset + nr);
return 0;
}
-static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
+static void __kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start,
+ pgoff_t end)
{
- struct address_space *mapping = inode->i_mapping;
- pgoff_t start, index, end;
- int r;
-
- /* Dedicated guest is immutable by default. */
- if (offset + len > i_size_read(inode))
- return -EINVAL;
-
- filemap_invalidate_lock_shared(mapping);
-
- start = offset >> PAGE_SHIFT;
- end = (offset + len) >> PAGE_SHIFT;
-
- r = 0;
- for (index = start; index < end; ) {
- struct folio *folio;
-
- if (signal_pending(current)) {
- r = -EINTR;
- break;
- }
-
- folio = kvm_gmem_get_folio(inode, index);
- if (IS_ERR(folio)) {
- r = PTR_ERR(folio);
- break;
- }
-
- index = folio_next_index(folio);
-
- folio_unlock(folio);
- folio_put(folio);
-
- /* 64-bit only, wrapping the index should be impossible. */
- if (WARN_ON_ONCE(!index))
- break;
+ struct kvm *kvm = gmem->kvm;
- cond_resched();
+ if (xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {
+ KVM_MMU_LOCK(kvm);
+ kvm_mmu_invalidate_end(kvm);
+ KVM_MMU_UNLOCK(kvm);
}
-
- filemap_invalidate_unlock_shared(mapping);
-
- return r;
}
-static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
- loff_t len)
+static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t offset, unsigned long nr)
{
- int ret;
-
- if (!(mode & FALLOC_FL_KEEP_SIZE))
- return -EOPNOTSUPP;
-
- if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
- return -EOPNOTSUPP;
-
- if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))
- return -EINVAL;
-
- if (mode & FALLOC_FL_PUNCH_HOLE)
- ret = kvm_gmem_punch_hole(file_inode(file), offset, len);
- else
- ret = kvm_gmem_allocate(file_inode(file), offset, len);
+ struct kvm_gmem *gmem;
- if (!ret)
- file_modified(file);
- return ret;
+ list_for_each_entry(gmem, &inode->i_mapping->i_private_list, entry)
+ __kvm_gmem_invalidate_end(gmem, offset, offset + nr);
}
-static int kvm_gmem_release(struct inode *inode, struct file *file)
+static void __kvm_gmem_release(struct inode *inode, struct kvm_gmem *gmem)
{
- struct kvm_gmem *gmem = file->private_data;
struct kvm_memory_slot *slot;
struct kvm *kvm = gmem->kvm;
unsigned long index;
@@ -274,8 +138,8 @@ static int kvm_gmem_release(struct inode *inode, struct file *file)
* Zap all SPTEs pointed at by this file. Do not free the backing
* memory, as its lifetime is associated with the inode, not the file.
*/
- kvm_gmem_invalidate_begin(gmem, 0, -1ul);
- kvm_gmem_invalidate_end(gmem, 0, -1ul);
+ __kvm_gmem_invalidate_begin(gmem, 0, -1ul);
+ __kvm_gmem_invalidate_end(gmem, 0, -1ul);
list_del(&gmem->entry);
@@ -287,6 +151,14 @@ static int kvm_gmem_release(struct inode *inode, struct file *file)
kfree(gmem);
kvm_put_kvm(kvm);
+}
+
+static int kvm_gmem_release(struct inode *inode)
+{
+ struct kvm_gmem *gmem;
+
+ list_for_each_entry(gmem, &inode->i_mapping->i_private_list, entry)
+ __kvm_gmem_release(inode, gmem);
return 0;
}
@@ -302,94 +174,11 @@ static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot)
return get_file_active(&slot->gmem.file);
}
-static struct file_operations kvm_gmem_fops = {
- .open = generic_file_open,
- .release = kvm_gmem_release,
- .fallocate = kvm_gmem_fallocate,
-};
-
-void kvm_gmem_init(struct module *module)
-{
- kvm_gmem_fops.owner = module;
-}
-
-static int kvm_gmem_migrate_folio(struct address_space *mapping,
- struct folio *dst, struct folio *src,
- enum migrate_mode mode)
-{
- WARN_ON_ONCE(1);
- return -EINVAL;
-}
-
-static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio)
-{
- struct list_head *gmem_list = &mapping->i_private_list;
- struct kvm_gmem *gmem;
- pgoff_t start, end;
-
- filemap_invalidate_lock_shared(mapping);
-
- start = folio->index;
- end = start + folio_nr_pages(folio);
-
- list_for_each_entry(gmem, gmem_list, entry)
- kvm_gmem_invalidate_begin(gmem, start, end);
-
- /*
- * Do not truncate the range, what action is taken in response to the
- * error is userspace's decision (assuming the architecture supports
- * gracefully handling memory errors). If/when the guest attempts to
- * access a poisoned page, kvm_gmem_get_pfn() will return -EHWPOISON,
- * at which point KVM can either terminate the VM or propagate the
- * error to userspace.
- */
-
- list_for_each_entry(gmem, gmem_list, entry)
- kvm_gmem_invalidate_end(gmem, start, end);
-
- filemap_invalidate_unlock_shared(mapping);
-
- return MF_DELAYED;
-}
-
-#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
-static void kvm_gmem_free_folio(struct folio *folio)
-{
- struct page *page = folio_page(folio, 0);
- kvm_pfn_t pfn = page_to_pfn(page);
- int order = folio_order(folio);
-
- kvm_arch_gmem_invalidate(pfn, pfn + (1ul << order));
-}
-#endif
-
-static const struct address_space_operations kvm_gmem_aops = {
- .dirty_folio = noop_dirty_folio,
- .migrate_folio = kvm_gmem_migrate_folio,
- .error_remove_folio = kvm_gmem_error_folio,
-#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
- .free_folio = kvm_gmem_free_folio,
-#endif
-};
-
-static int kvm_gmem_getattr(struct mnt_idmap *idmap, const struct path *path,
- struct kstat *stat, u32 request_mask,
- unsigned int query_flags)
-{
- struct inode *inode = path->dentry->d_inode;
-
- generic_fillattr(idmap, request_mask, inode, stat);
- return 0;
-}
-
-static int kvm_gmem_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
- struct iattr *attr)
-{
- return -EINVAL;
-}
-static const struct inode_operations kvm_gmem_iops = {
- .getattr = kvm_gmem_getattr,
- .setattr = kvm_gmem_setattr,
+static const struct guest_memfd_operations kvm_gmem_ops = {
+ .invalidate_begin = kvm_gmem_invalidate_begin,
+ .invalidate_end = kvm_gmem_invalidate_end,
+ .prepare_inaccessible = kvm_gmem_prepare_inaccessible,
+ .release = kvm_gmem_release,
};
static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
@@ -410,28 +199,16 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
goto err_fd;
}
- file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem,
- O_RDWR, NULL);
+ file = guest_memfd_alloc(anon_name, &kvm_gmem_ops, size,
+ GUEST_MEMFD_FLAG_CLEAR_INACCESSIBLE);
if (IS_ERR(file)) {
err = PTR_ERR(file);
goto err_gmem;
}
- file->f_flags |= O_LARGEFILE;
-
inode = file->f_inode;
WARN_ON(file->f_mapping != inode->i_mapping);
- inode->i_private = (void *)(unsigned long)flags;
- inode->i_op = &kvm_gmem_iops;
- inode->i_mapping->a_ops = &kvm_gmem_aops;
- inode->i_mode |= S_IFREG;
- inode->i_size = size;
- mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
- mapping_set_inaccessible(inode->i_mapping);
- /* Unmovable mappings are supposed to be marked unevictable as well. */
- WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));
-
kvm_get_kvm(kvm);
gmem->kvm = kvm;
xa_init(&gmem->bindings);
@@ -462,6 +239,14 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
return __kvm_gmem_create(kvm, size, flags);
}
+static inline struct kvm_gmem *file_to_kvm_gmem(struct file *file)
+{
+ if (!is_guest_memfd(file, &kvm_gmem_ops))
+ return NULL;
+
+ return inode_to_kvm_gmem(file_inode(file));
+}
+
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
unsigned int fd, loff_t offset)
{
@@ -478,11 +263,8 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
if (!file)
return -EBADF;
- if (file->f_op != &kvm_gmem_fops)
- goto err;
-
- gmem = file->private_data;
- if (gmem->kvm != kvm)
+ gmem = file_to_kvm_gmem(file);
+ if (!gmem || gmem->kvm != kvm)
goto err;
inode = file_inode(file);
@@ -539,7 +321,9 @@ void kvm_gmem_unbind(struct kvm_memory_slot *slot)
if (!file)
return;
- gmem = file->private_data;
+ gmem = file_to_kvm_gmem(file);
+ if (WARN_ON_ONCE(!gmem))
+ return;
filemap_invalidate_lock(file->f_mapping);
xa_store_range(&gmem->bindings, start, end - 1, NULL, GFP_KERNEL);
@@ -553,7 +337,7 @@ void kvm_gmem_unbind(struct kvm_memory_slot *slot)
/* Returns a locked folio on success. */
static struct folio *
__kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,
- gfn_t gfn, kvm_pfn_t *pfn, bool *is_prepared,
+ gfn_t gfn, kvm_pfn_t *pfn, bool accessible,
int *max_order)
{
pgoff_t index = gfn - slot->base_gfn + slot->gmem.pgoff;
@@ -565,19 +349,27 @@ __kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,
return ERR_PTR(-EFAULT);
}
- gmem = file->private_data;
+ gmem = file_to_kvm_gmem(file);
+ if (WARN_ON_ONCE(!gmem))
+ return ERR_PTR(-EFAULT);
+
if (xa_load(&gmem->bindings, index) != slot) {
WARN_ON_ONCE(xa_load(&gmem->bindings, index));
return ERR_PTR(-EIO);
}
- folio = kvm_gmem_get_folio(file_inode(file), index);
+ if (accessible)
+ grab_flags = GUEST_MEMFD_GRAB_ACCESSIBLE;
+ else
+ grab_flags = GUEST_MEMFD_GRAB_INACCESSIBLE;
+
+ folio = guest_memfd_grab_folio(file, index, grab_flags);
if (IS_ERR(folio))
return folio;
if (folio_test_hwpoison(folio)) {
folio_unlock(folio);
- folio_put(folio);
+ guest_memfd_put_folio(folio, accessible ? 1 : 0);
return ERR_PTR(-EHWPOISON);
}
@@ -585,7 +377,6 @@ __kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,
if (max_order)
*max_order = 0;
- *is_prepared = folio_test_uptodate(folio);
return folio;
}
@@ -594,24 +385,22 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
{
struct file *file = kvm_gmem_get_file(slot);
struct folio *folio;
- bool is_prepared = false;
int r = 0;
if (!file)
return -EFAULT;
- folio = __kvm_gmem_get_pfn(file, slot, gfn, pfn, &is_prepared, max_order);
+ folio = __kvm_gmem_get_pfn(file, slot, gfn, pfn, false, max_order);
if (IS_ERR(folio)) {
r = PTR_ERR(folio);
goto out;
}
- if (!is_prepared)
- r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);
-
folio_unlock(folio);
if (r < 0)
- folio_put(folio);
+ guest_memfd_put_folio(folio, 0);
+ else
+ guest_memfd_unsafe_folio(folio);
out:
fput(file);
@@ -648,7 +437,6 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long
for (i = 0; i < npages; i += (1 << max_order)) {
struct folio *folio;
gfn_t gfn = start_gfn + i;
- bool is_prepared = false;
kvm_pfn_t pfn;
if (signal_pending(current)) {
@@ -656,19 +444,12 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long
break;
}
- folio = __kvm_gmem_get_pfn(file, slot, gfn, &pfn, &is_prepared, &max_order);
+ folio = __kvm_gmem_get_pfn(file, slot, gfn, &pfn, true, &max_order);
if (IS_ERR(folio)) {
ret = PTR_ERR(folio);
break;
}
- if (is_prepared) {
- folio_unlock(folio);
- folio_put(folio);
- ret = -EEXIST;
- break;
- }
-
folio_unlock(folio);
WARN_ON(!IS_ALIGNED(gfn, 1 << max_order) ||
(npages - i) < (1 << max_order));
@@ -684,11 +465,9 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long
p = src ? src + i * PAGE_SIZE : NULL;
ret = post_populate(kvm, gfn, pfn, p, max_order, opaque);
- if (!ret)
- kvm_gmem_mark_prepared(folio);
put_folio_and_exit:
- folio_put(folio);
+ guest_memfd_put_folio(folio, 1);
if (ret)
break;
}
@@ -6514,8 +6514,6 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
if (WARN_ON_ONCE(r))
goto err_vfio;
- kvm_gmem_init(module);
-
/*
* Registration _must_ be the very last thing done, as this exposes
* /dev/kvm to userspace, i.e. all infrastructure must be setup!
@@ -36,17 +36,11 @@ static inline void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm,
#endif /* HAVE_KVM_PFNCACHE */
#ifdef CONFIG_KVM_PRIVATE_MEM
-void kvm_gmem_init(struct module *module);
int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args);
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
unsigned int fd, loff_t offset);
void kvm_gmem_unbind(struct kvm_memory_slot *slot);
#else
-static inline void kvm_gmem_init(struct module *module)
-{
-
-}
-
static inline int kvm_gmem_bind(struct kvm *kvm,
struct kvm_memory_slot *slot,
unsigned int fd, loff_t offset)
Use the recently created mm/guest_memfd implementation. No functional change intended. Note: I've only compile-tested this. Appreciate some help from SEV folks to be able to test this. Signed-off-by: Elliot Berman <quic_eberman@quicinc.com> --- arch/x86/kvm/svm/sev.c | 3 +- virt/kvm/Kconfig | 1 + virt/kvm/guest_memfd.c | 371 ++++++++++--------------------------------------- virt/kvm/kvm_main.c | 2 - virt/kvm/kvm_mm.h | 6 - 5 files changed, 77 insertions(+), 306 deletions(-)