106 lines
2.4 KiB
C
106 lines
2.4 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* accelerate copying page to pmem with non-temproal stroes
|
|
*/
|
|
#include <linux/sched.h>
|
|
#include <linux/mmzone.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/sysctl.h>
|
|
|
|
DEFINE_STATIC_KEY_FALSE(hugepage_nocache_copy);
|
|
#ifdef CONFIG_SYSCTL
|
|
static void set_hugepage_nocache_copy(bool enabled)
|
|
{
|
|
if (enabled)
|
|
static_branch_enable(&hugepage_nocache_copy);
|
|
else
|
|
static_branch_disable(&hugepage_nocache_copy);
|
|
}
|
|
|
|
int sysctl_hugepage_nocache_copy(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp, loff_t *ppos)
|
|
{
|
|
struct ctl_table t;
|
|
int err;
|
|
int state;
|
|
|
|
if (write && !capable(CAP_SYS_ADMIN))
|
|
return -EPERM;
|
|
|
|
state = static_branch_unlikely(&hugepage_nocache_copy);
|
|
t = *table;
|
|
t.data = &state;
|
|
err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
|
|
if (err < 0)
|
|
return err;
|
|
if (write)
|
|
set_hugepage_nocache_copy(state);
|
|
return err;
|
|
}
|
|
|
|
static struct ctl_table copy_highpages_table[] = {
|
|
{
|
|
.procname = "hugepage_nocache_copy",
|
|
.data = NULL,
|
|
.maxlen = sizeof(unsigned int),
|
|
.mode = 0600,
|
|
.proc_handler = sysctl_hugepage_nocache_copy,
|
|
.extra1 = SYSCTL_ZERO,
|
|
.extra2 = SYSCTL_ONE,
|
|
},
|
|
{}
|
|
};
|
|
|
|
static struct ctl_table copy_highpages_root_table[] = {
|
|
{
|
|
.procname = "vm",
|
|
.mode = 0555,
|
|
.child = copy_highpages_table,
|
|
},
|
|
{}
|
|
};
|
|
|
|
static __init int copy_highpages_init(void)
|
|
{
|
|
return register_sysctl_table(copy_highpages_root_table) ? 0 : -ENOMEM;
|
|
}
|
|
__initcall(copy_highpages_init);
|
|
#endif
|
|
|
|
static void copy_highpages_nocache(struct page *to, struct page *from, int nr_pages)
|
|
{
|
|
char *vfrom, *vto;
|
|
int i;
|
|
|
|
for (i = 0; i < nr_pages; i++) {
|
|
cond_resched();
|
|
vfrom = kmap_atomic(from);
|
|
vto = kmap_atomic(to);
|
|
copy_page_nocache(vto, vfrom);
|
|
kunmap_atomic(vto);
|
|
kunmap_atomic(vfrom);
|
|
to++;
|
|
from++;
|
|
}
|
|
copy_page_nocache_barrir();
|
|
}
|
|
|
|
static void copy_highpages_cache(struct page *to, struct page *from, int nr_pages)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < nr_pages; i++) {
|
|
cond_resched();
|
|
copy_highpage(to + i, from + i);
|
|
}
|
|
}
|
|
|
|
void copy_highpages(struct page *to, struct page *from, int nr_pages)
|
|
{
|
|
if (static_branch_unlikely(&hugepage_nocache_copy) &&
|
|
get_node_type(page_to_nid(to)) == NODE_TYPE_PMEM)
|
|
return copy_highpages_nocache(to, from, nr_pages);
|
|
|
|
return copy_highpages_cache(to, from, nr_pages);
|
|
}
|