Skip to content

Commit

Permalink
Followup on r302393 by cperciva, improving calculation of boot pages …
Browse files Browse the repository at this point in the history 
…required

for UMA startup.

o Introduce another stage of UMA startup, which is entered after
  vm_page_startup() finishes. After this stage we don't yet enable buckets,
  but we can ask VM for pages. Rename stages to meaningful names while here.
  New list of stages: BOOT_COLD, BOOT_STRAPPED, BOOT_PAGEALLOC, BOOT_BUCKETS,
  BOOT_RUNNING.
  Enabling page alloc earlier allows us to dramatically reduce number of
  boot pages required. What is more important number of zones becomes
  consistent across different machines, as no MD allocations are done before
  the BOOT_PAGEALLOC stage. Now only UMA internal zones actually need to use
  startup_alloc(), however that may change, so vm_page_startup() provides
  its need for early zones as argument.
o Introduce uma_startup_count() function, to avoid code duplication. The
  functions calculates sizes of zones zone and kegs zone, and calculates how
  many pages UMA will need to bootstrap.
  It counts not only of zone structures, but also of kegs, slabs and hashes.
o Hide uma_startup_foo() declarations from public file.
o Provide several DIAGNOSTIC printfs on boot_pages usage.
o Bugfix: when calculating zone of zones size use (mp_maxid + 1) instead of
  mp_ncpus. Use resulting number not only in the size argument to zone_ctor()
  but also as args.size.

Reviewed by:		imp, gallatin (earlier version)
Differential Revision:	https://reviews.freebsd.org/D14054
  • Loading branch information
@glebius
glebius committed Feb 6, 2018
1 parent a7ac77d commit baecd1a
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 79 deletions.
2 changes: 2 additions & 0 deletions sys/kern/kern_malloc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ __FBSDID("$FreeBSD$");
dtrace_malloc_probe_func_t dtrace_malloc_probe;
#endif

extern void uma_startup2(void);

#if defined(INVARIANTS) || defined(MALLOC_MAKE_FAILURES) || \
defined(DEBUG_MEMGUARD) || defined(DEBUG_REDZONE)
#define MALLOC_DEBUG 1
Expand Down
34 changes: 0 additions & 34 deletions sys/vm/uma.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,40 +430,6 @@ typedef void *(*uma_alloc)(uma_zone_t zone, vm_size_t size, int domain,
*/
typedef void (*uma_free)(void *item, vm_size_t size, uint8_t pflag);

/*
* Sets up the uma allocator. (Called by vm_mem_init)
*
* Arguments:
* bootmem A pointer to memory used to bootstrap the system.
*
* Returns:
* Nothing
*
* Discussion:
* This memory is used for zones which allocate things before the
* backend page supplier can give us pages. It should be
* UMA_SLAB_SIZE * boot_pages bytes. (see uma_int.h)
*
*/

void uma_startup(void *bootmem, int boot_pages);

/*
* Finishes starting up the allocator. This should
* be called when kva is ready for normal allocs.
*
* Arguments:
* None
*
* Returns:
* Nothing
*
* Discussion:
* uma_startup2 is called by kmeminit() to enable us of uma for malloc.
*/

void uma_startup2(void);

/*
* Reclaims unused memory for all zones
*
Expand Down
106 changes: 81 additions & 25 deletions sys/vm/uma_core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,8 @@ static unsigned long uma_kmem_limit = LONG_MAX;
static volatile unsigned long uma_kmem_total;

/* Is the VM done starting up? */
static int booted = 0;
#define UMA_STARTUP 1
#define UMA_STARTUP2 2
static enum { BOOT_COLD = 0, BOOT_STRAPPED, BOOT_PAGEALLOC, BOOT_BUCKETS,
BOOT_RUNNING } booted = BOOT_COLD;

/*
* This is the handle used to schedule events that need to happen
Expand Down Expand Up @@ -226,6 +225,11 @@ enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };

/* Prototypes.. */

int uma_startup_count(int);
void uma_startup(void *, int);
void uma_startup1(void);
void uma_startup2(void);

static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
Expand Down Expand Up @@ -1084,6 +1088,11 @@ startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
* Check our small startup cache to see if it has pages remaining.
*/
mtx_lock(&uma_boot_pages_mtx);
#ifdef DIAGNOSTIC
if (booted < BOOT_PAGEALLOC)
printf("%s from \"%s\", %d boot pages left\n", __func__,
zone->uz_name, boot_pages);
#endif
if (pages <= boot_pages) {
mem = bootmem;
boot_pages -= pages;
Expand All @@ -1093,7 +1102,7 @@ startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
return (mem);
}
mtx_unlock(&uma_boot_pages_mtx);
if (booted < UMA_STARTUP2)
if (booted < BOOT_PAGEALLOC)
panic("UMA: Increase vm.boot_pages");
/*
* Now that we've booted reset these users to their real allocator.
Expand Down Expand Up @@ -1472,7 +1481,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
* If we haven't booted yet we need allocations to go through the
* startup cache until the vm is ready.
*/
if (booted < UMA_STARTUP2)
if (booted < BOOT_PAGEALLOC)
keg->uk_allocf = startup_alloc;
#ifdef UMA_MD_SMALL_ALLOC
else if (keg->uk_ppera == 1)
Expand Down Expand Up @@ -1770,24 +1779,62 @@ zone_foreach(void (*zfunc)(uma_zone_t))
rw_runlock(&uma_rwlock);
}

/* Public functions */
/* See uma.h */
/*
* Count how many pages do we need to bootstrap. VM supplies
* its need in early zones in the argument, we add up our zones,
* which consist of: UMA Slabs, UMA Hash and 9 Bucket zones. The
* zone of zones and zone of kegs are accounted separately.
*/
#define UMA_BOOT_ZONES 11
static int zsize, ksize;
int
uma_startup_count(int zones)
{
int pages;

ksize = sizeof(struct uma_keg) +
(sizeof(struct uma_domain) * vm_ndomains);
zsize = sizeof(struct uma_zone) +
(sizeof(struct uma_cache) * (mp_maxid + 1)) +
(sizeof(struct uma_zone_domain) * vm_ndomains);

/* Memory for the zone of zones and zone of kegs. */
pages = howmany(roundup(zsize, CACHE_LINE_SIZE) * 2 +
roundup(ksize, CACHE_LINE_SIZE), PAGE_SIZE);

zones += UMA_BOOT_ZONES;

/* Memory for startup zones, UMA and VM, ... */
if (zsize > UMA_SLAB_SIZE)
pages += zones * howmany(zsize, UMA_SLAB_SIZE);
else
pages += howmany(zones, UMA_SLAB_SIZE / zsize);

/* ... and their kegs. */
pages += howmany(ksize * zones, UMA_SLAB_SIZE);

/*
* Take conservative approach that every zone
* is going to allocate hash.
*/
pages += howmany(sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT *
zones, UMA_SLAB_SIZE);

return (pages);
}

void
uma_startup(void *mem, int npages)
{
struct uma_zctor_args args;
uma_keg_t masterkeg;
uintptr_t m;
int zsize;
int ksize;

rw_init(&uma_rwlock, "UMA lock");
#ifdef DIAGNOSTIC
printf("Entering %s with %d boot pages configured\n", __func__, npages);
#endif

ksize = sizeof(struct uma_keg) +
(sizeof(struct uma_domain) * vm_ndomains);
zsize = sizeof(struct uma_zone) +
(sizeof(struct uma_cache) * mp_ncpus) +
(sizeof(struct uma_zone_domain) * vm_ndomains);
rw_init(&uma_rwlock, "UMA lock");

/* Use bootpages memory for the zone of zones and zone of kegs. */
m = (uintptr_t)mem;
Expand Down Expand Up @@ -1819,9 +1866,7 @@ uma_startup(void *mem, int npages)
boot_pages = npages;

args.name = "UMA Zones";
args.size = sizeof(struct uma_zone) +
(sizeof(struct uma_cache) * (mp_maxid + 1)) +
(sizeof(struct uma_zone_domain) * vm_ndomains);
args.size = zsize;
args.ctor = zone_ctor;
args.dtor = zone_dtor;
args.uminit = zero_init;
Expand All @@ -1844,27 +1889,37 @@ uma_startup(void *mem, int npages)

bucket_init();

booted = UMA_STARTUP;
booted = BOOT_STRAPPED;
}

void
uma_startup1(void)
{

#ifdef DIAGNOSTIC
printf("Entering %s with %d boot pages left\n", __func__, boot_pages);
#endif
booted = BOOT_PAGEALLOC;
}

/* see uma.h */
void
uma_startup2(void)
{
booted = UMA_STARTUP2;
bucket_enable();

booted = BOOT_BUCKETS;
sx_init(&uma_drain_lock, "umadrain");
bucket_enable();
}

/*
* Initialize our callout handle
*
*/

static void
uma_startup3(void)
{

booted = BOOT_RUNNING;
callout_init(&uma_callout, 1);
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
}
Expand All @@ -1884,6 +1939,7 @@ uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK));
}

/* Public functions */
/* See uma.h */
void
uma_set_align(int align)
Expand Down Expand Up @@ -1932,7 +1988,7 @@ uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
args.flags = flags;
args.keg = NULL;

if (booted < UMA_STARTUP2) {
if (booted < BOOT_BUCKETS) {
locked = false;
} else {
sx_slock(&uma_drain_lock);
Expand Down Expand Up @@ -1966,7 +2022,7 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
args.keg = keg;

if (booted < UMA_STARTUP2) {
if (booted < BOOT_BUCKETS) {
locked = false;
} else {
sx_slock(&uma_drain_lock);
Expand Down
4 changes: 0 additions & 4 deletions sys/vm/uma_int.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,6 @@
#define UMA_SLAB_MASK (PAGE_SIZE - 1) /* Mask to get back to the page */
#define UMA_SLAB_SHIFT PAGE_SHIFT /* Number of bits PAGE_MASK */

#define UMA_BOOT_PAGES 64 /* Pages allocated for startup */
#define UMA_BOOT_PAGES_ZONES 32 /* Multiplier for pages to reserve */
/* if uma_zone > PAGE_SIZE */

/* Max waste percentage before going to off page slab management */
#define UMA_MAX_WASTE 10

Expand Down
26 changes: 10 additions & 16 deletions sys/vm/vm_page.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ __FBSDID("$FreeBSD$");

#include <machine/md_var.h>

extern int uma_startup_count(int);
extern void uma_startup(void *, int);
extern void uma_startup1(void);

/*
* Associated with page of user-allocatable memory is a
* page structure.
Expand All @@ -145,7 +149,7 @@ vm_page_t vm_page_array;
long vm_page_array_size;
long first_page;

static int boot_pages = UMA_BOOT_PAGES;
static int boot_pages;
SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
&boot_pages, 0,
"number of pages allocated for bootstrapping the VM system");
Expand Down Expand Up @@ -466,7 +470,7 @@ vm_page_startup(vm_offset_t vaddr)
vm_paddr_t end, high_avail, low_avail, new_end, page_range, size;
vm_paddr_t biggestsize, last_pa, pa;
u_long pagecount;
int biggestone, i, pages_per_zone, segind;
int biggestone, i, segind;

biggestsize = 0;
biggestone = 0;
Expand Down Expand Up @@ -495,27 +499,14 @@ vm_page_startup(vm_offset_t vaddr)
for (i = 0; i < vm_ndomains; i++)
vm_page_domain_init(&vm_dom[i]);

/*
* Almost all of the pages needed for bootstrapping UMA are used
* for zone structures, so if the number of CPUs results in those
* structures taking more than one page each, we set aside more pages
* in proportion to the zone structure size.
*/
pages_per_zone = howmany(sizeof(struct uma_zone) +
sizeof(struct uma_cache) * (mp_maxid + 1) +
roundup2(sizeof(struct uma_slab), sizeof(void *)), UMA_SLAB_SIZE);
if (pages_per_zone > 1) {
/* Reserve more pages so that we don't run out. */
boot_pages = UMA_BOOT_PAGES_ZONES * pages_per_zone;
}

/*
* Allocate memory for use when boot strapping the kernel memory
* allocator.
*
* CTFLAG_RDTUN doesn't work during the early boot process, so we must
* manually fetch the value.
*/
boot_pages = uma_startup_count(0);
TUNABLE_INT_FETCH("vm.boot_pages", &boot_pages);
new_end = end - (boot_pages * UMA_SLAB_SIZE);
new_end = trunc_page(new_end);
Expand Down Expand Up @@ -749,6 +740,9 @@ vm_page_startup(vm_offset_t vaddr)
*/
domainset_zero();

/* Announce page availability to UMA. */
uma_startup1();

return (vaddr);
}

Expand Down

0 comments on commit baecd1a

Please sign in to comment.