@@ -18,6 +18,17 @@
odp_implementation = "linux-generic"
config_file_version = "0.0.1"
+# Internal shared memory allocator
+shm: {
+ # ODP will try to reserve as many huge pages as the number indicated
+ # here, up to 64. A zero value means that no pages should be reserved.
+ # When using process mode threads, this value should be set to 0
+ # because the current implementation won't work properly otherwise.
+ # These pages will only be freed when the application calls
+ # odp_term_global().
+ num_cached_hp = 0
+}
+
# DPDK pktio options
pktio_dpdk: {
# Default options
@@ -63,6 +63,7 @@
#include <odp_ishm_internal.h>
#include <odp_ishmphy_internal.h>
#include <odp_ishmpool_internal.h>
+#include <odp_libconfig_internal.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
@@ -164,7 +165,7 @@ typedef struct ishm_fragment {
* will allocate both a block and a fragment.
* Blocks contain only global data common to all processes.
*/
-typedef enum {UNKNOWN, HUGE, NORMAL, EXTERNAL} huge_flag_t;
+typedef enum {UNKNOWN, HUGE, NORMAL, EXTERNAL, CACHED} huge_flag_t;
typedef struct ishm_block {
char name[ISHM_NAME_MAXLEN]; /* name for the ishm block (if any) */
char filename[ISHM_FILENAME_MAXLEN]; /* name of the .../odp-* file */
@@ -238,6 +239,16 @@ typedef struct {
} ishm_ftable_t;
static ishm_ftable_t *ishm_ftbl;
+#define HP_CACHE_SIZE 64
+struct huge_page_cache {
+ uint64_t len;
+ int total; /* amount of actually pre-allocated huge pages */
+ int idx; /* retrieve fd[idx] to get a free file descriptor */
+ int fd[HP_CACHE_SIZE]; /* list of file descriptors */
+};
+
+static struct huge_page_cache hpc;
+
#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif
@@ -245,6 +256,142 @@ static ishm_ftable_t *ishm_ftbl;
/* prototypes: */
static void procsync(void);
+static int hp_create_file(uint64_t len, const char *filename)
+{
+ int fd;
+ void *addr;
+
+ if (len <= 0) {
+ ODP_ERR("Length is wrong\n");
+ return -1;
+ }
+
+ fd = open(filename, O_RDWR | O_CREAT | O_TRUNC,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+ if (fd < 0) {
+ ODP_ERR("Could not create cache file %s\n", filename);
+ return -1;
+ }
+
+ /* remove file from file system */
+ unlink(filename);
+
+ if (ftruncate(fd, len) == -1) {
+ ODP_ERR("Could not truncate file: %s\n", strerror(errno));
+ close(fd);
+ return -1;
+ }
+
+ /* commit huge page */
+ addr = _odp_ishmphy_map(fd, NULL, len, 0);
+ if (addr == NULL) {
+ /* no more pages available */
+ close(fd);
+ return -1;
+ }
+ _odp_ishmphy_unmap(addr, len, 0);
+
+ ODP_DBG("Created HP cache file %s, fd: %d\n", filename, fd);
+
+ return fd;
+}
+
+static void hp_init(void)
+{
+ char filename[ISHM_FILENAME_MAXLEN];
+ char dir[ISHM_FILENAME_MAXLEN];
+ int count;
+
+ hpc.total = 0;
+ hpc.idx = -1;
+ hpc.len = odp_sys_huge_page_size();
+
+ if (!_odp_libconfig_lookup_ext_int("shm", NULL, "num_cached_hp",
+ &count)) {
+ return;
+ }
+
+ if (count > HP_CACHE_SIZE)
+ count = HP_CACHE_SIZE;
+ else if (count <= 0)
+ return;
+
+ ODP_DBG("Init HP cache with up to %d pages\n", count);
+
+ if (!odp_global_data.hugepage_info.default_huge_page_dir) {
+ ODP_ERR("No huge page dir\n");
+ return;
+ }
+
+ snprintf(dir, ISHM_FILENAME_MAXLEN, "%s/%s",
+ odp_global_data.hugepage_info.default_huge_page_dir,
+ odp_global_data.uid);
+
+ if (mkdir(dir, 0744) != 0) {
+ if (errno != EEXIST) {
+ ODP_ERR("Failed to create dir: %s\n", strerror(errno));
+ return;
+ }
+ }
+
+ snprintf(filename, ISHM_FILENAME_MAXLEN,
+ "%s/odp-%d-ishm_cached",
+ dir,
+ odp_global_data.main_pid);
+
+ for (int i = 0; i < count; ++i) {
+ int fd;
+
+ fd = hp_create_file(hpc.len, filename);
+ if (fd == -1)
+ break;
+ hpc.total++;
+ hpc.fd[i] = fd;
+ }
+ hpc.idx = hpc.total - 1;
+
+ ODP_DBG("HP cache has %d huge pages of size 0x%08" PRIx64 "\n",
+ hpc.total, hpc.len);
+}
+
+static void hp_term(void)
+{
+ for (int i = 0; i < hpc.total; i++) {
+ if (hpc.fd[i] != -1)
+ close(hpc.fd[i]);
+ }
+
+ hpc.total = 0;
+ hpc.idx = -1;
+ hpc.len = 0;
+}
+
+static int hp_get_cached(uint64_t len)
+{
+ int fd;
+
+ if (hpc.idx < 0 || len != hpc.len)
+ return -1;
+
+ fd = hpc.fd[hpc.idx];
+ hpc.fd[hpc.idx--] = -1;
+
+ return fd;
+}
+
+static int hp_put_cached(int fd)
+{
+ if (odp_unlikely(++hpc.idx >= hpc.total)) {
+ hpc.idx--;
+ ODP_ERR("Trying to put more FD than allowed: %d\n", fd);
+ return -1;
+ }
+
+ hpc.fd[hpc.idx] = fd;
+
+ return 0;
+}
+
/*
* Take a piece of the preallocated virtual space to fit "size" bytes.
* (best fit). Size must be rounded up to an integer number of pages size.
@@ -798,8 +945,14 @@ static int block_free_internal(int block_index, int close_fd, int deregister)
block_index);
/* close the related fd */
- if (close_fd)
- close(ishm_proctable->entry[proc_index].fd);
+ if (close_fd) {
+ int fd = ishm_proctable->entry[proc_index].fd;
+
+ if (block->huge == CACHED)
+ hp_put_cached(fd);
+ else
+ close(fd);
+ }
/* remove entry from process local table: */
last = ishm_proctable->nb_entries - 1;
@@ -910,6 +1063,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd,
new_block->huge = EXTERNAL;
} else {
new_block->external_fd = 0;
+ new_block->huge = UNKNOWN;
}
/* Otherwise, Try first huge pages when possible and needed: */
@@ -927,17 +1081,38 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd,
/* roundup to page size */
len = (size + (page_hp_size - 1)) & (-page_hp_size);
- addr = do_map(new_index, len, hp_align, flags, HUGE, &fd);
-
- if (addr == NULL) {
- if (!huge_error_printed) {
- ODP_ERR("No huge pages, fall back to normal "
- "pages. "
- "check: /proc/sys/vm/nr_hugepages.\n");
- huge_error_printed = 1;
+ if (!(flags & _ODP_ISHM_SINGLE_VA)) {
+ /* try pre-allocated pages */
+ fd = hp_get_cached(len);
+ if (fd != -1) {
+ /* do as if user provided a fd */
+ new_block->external_fd = 1;
+ addr = do_map(new_index, len, hp_align, flags,
+ CACHED, &fd);
+ if (addr == NULL) {
+ ODP_ERR("Could not use cached hp %d\n",
+ fd);
+ hp_put_cached(fd);
+ fd = -1;
+ } else {
+ new_block->huge = CACHED;
+ }
+ }
+ }
+ if (fd == -1) {
+ addr = do_map(new_index, len, hp_align, flags, HUGE,
+ &fd);
+
+ if (addr == NULL) {
+ if (!huge_error_printed) {
+ ODP_ERR("No huge pages, fall back to "
+ "normal pages. Check: "
+ "/proc/sys/vm/nr_hugepages.\n");
+ huge_error_printed = 1;
+ }
+ } else {
+ new_block->huge = HUGE;
}
- } else {
- new_block->huge = HUGE;
}
}
@@ -961,8 +1136,12 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd,
/* if neither huge pages or normal pages works, we cannot proceed: */
if ((fd < 0) || (addr == NULL) || (len == 0)) {
- if ((!new_block->external_fd) && (fd >= 0))
+ if (new_block->external_fd) {
+ if (new_block->huge == CACHED)
+ hp_put_cached(fd);
+ } else if (fd >= 0) {
close(fd);
+ }
delete_file(new_block);
odp_spinlock_unlock(&ishm_tbl->lock);
ODP_ERR("_ishm_reserve failed.\n");
@@ -1564,6 +1743,9 @@ int _odp_ishm_init_global(const odp_init_t *init)
/* get ready to create pools: */
_odp_ishm_pool_init();
+ /* init cache files */
+ hp_init();
+
return 0;
init_glob_err4:
@@ -1705,6 +1887,8 @@ int _odp_ishm_term_global(void)
if (!odp_global_data.shm_dir_from_env)
free(odp_global_data.shm_dir);
+ hp_term();
+
return ret;
}
@@ -1778,6 +1962,9 @@ int _odp_ishm_status(const char *title)
case EXTERNAL:
huge = 'E';
break;
+ case CACHED:
+ huge = 'C';
+ break;
default:
huge = '?';
}
@@ -1911,6 +2098,9 @@ void _odp_ishm_print(int block_index)
case EXTERNAL:
str = "external";
break;
+ case CACHED:
+ str = "cached";
+ break;
default:
str = "??";
}