From owner-svn-src-head@freebsd.org Wed May 31 16:08:31 2017 Return-Path: Delivered-To: svn-src-head@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 7893CB7D798; Wed, 31 May 2017 16:08:31 +0000 (UTC) (envelope-from hselasky@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 4D2EA8164F; Wed, 31 May 2017 16:08:31 +0000 (UTC) (envelope-from hselasky@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v4VG8UpP015136; Wed, 31 May 2017 16:08:30 GMT (envelope-from hselasky@FreeBSD.org) Received: (from hselasky@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v4VG8Uuk015134; Wed, 31 May 2017 16:08:30 GMT (envelope-from hselasky@FreeBSD.org) Message-Id: <201705311608.v4VG8Uuk015134@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: hselasky set sender to hselasky@FreeBSD.org using -f From: Hans Petter Selasky Date: Wed, 31 May 2017 16:08:30 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r319340 - in head/sys/compat/linuxkpi/common: include/linux src X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 31 May 2017 16:08:31 -0000 Author: hselasky Date: Wed May 31 16:08:30 2017 New Revision: 319340 URL: https://svnweb.freebsd.org/changeset/base/319340 Log: Properly implement idr_preload() and idr_preload_end() in the LinuxKPI. MFC after: 1 week Sponsored by: Mellanox Technologies Modified: head/sys/compat/linuxkpi/common/include/linux/idr.h head/sys/compat/linuxkpi/common/src/linux_idr.c Modified: head/sys/compat/linuxkpi/common/include/linux/idr.h ============================================================================== --- head/sys/compat/linuxkpi/common/include/linux/idr.h Wed May 31 16:07:32 2017 (r319339) +++ head/sys/compat/linuxkpi/common/include/linux/idr.h Wed May 31 16:08:30 2017 (r319340) @@ -75,9 +75,8 @@ struct idr { SYSINIT(name##_ida_sysinit, SI_SUB_DRIVERS, SI_ORDER_FIRST, \ ida_init, &(name)) -#define idr_preload(x) do { } while (0) -#define idr_preload_end() do { } while (0) - +void idr_preload(gfp_t gfp_mask); +void idr_preload_end(void); void *idr_find(struct idr *idp, int id); void *idr_get_next(struct idr *idp, int *nextid); int idr_pre_get(struct idr *idp, gfp_t gfp_mask); Modified: head/sys/compat/linuxkpi/common/src/linux_idr.c ============================================================================== --- head/sys/compat/linuxkpi/common/src/linux_idr.c Wed May 31 16:07:32 2017 (r319339) +++ head/sys/compat/linuxkpi/common/src/linux_idr.c Wed May 31 16:08:30 2017 (r319340) @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -46,6 +46,17 @@ __FBSDID("$FreeBSD$"); #include #include +#define MAX_IDR_LEVEL ((MAX_IDR_SHIFT + IDR_BITS - 1) / IDR_BITS) +#define MAX_IDR_FREE (MAX_IDR_LEVEL * 2) + +struct linux_idr_cache { + spinlock_t lock; + struct idr_layer *head; + unsigned count; +}; + +static DPCPU_DEFINE(struct linux_idr_cache, linux_idr_cache); + /* * IDR Implementation. * @@ -55,6 +66,96 @@ __FBSDID("$FreeBSD$"); */ static MALLOC_DEFINE(M_IDR, "idr", "Linux IDR compat"); +static struct idr_layer * +idr_preload_dequeue_locked(struct linux_idr_cache *lic) +{ + struct idr_layer *retval; + + /* check if wrong thread is trying to dequeue */ + if (mtx_owned(&lic->lock.m) == 0) + return (NULL); + + retval = lic->head; + if (likely(retval != NULL)) { + lic->head = retval->ary[0]; + lic->count--; + retval->ary[0] = NULL; + } + return (retval); +} + +static void +idr_preload_init(void *arg) +{ + int cpu; + + CPU_FOREACH(cpu) { + struct linux_idr_cache *lic = + DPCPU_ID_PTR(cpu, linux_idr_cache); + + spin_lock_init(&lic->lock); + } +} +SYSINIT(idr_preload_init, SI_SUB_LOCK, SI_ORDER_FIRST, idr_preload_init, NULL); + +static void +idr_preload_uninit(void *arg) +{ + int cpu; + + CPU_FOREACH(cpu) { + struct idr_layer *cacheval; + struct linux_idr_cache *lic = + DPCPU_ID_PTR(cpu, linux_idr_cache); + + while (1) { + spin_lock(&lic->lock); + cacheval = idr_preload_dequeue_locked(lic); + spin_unlock(&lic->lock); + + if (cacheval == NULL) + break; + free(cacheval, M_IDR); + } + spin_lock_destroy(&lic->lock); + } +} +SYSUNINIT(idr_preload_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, idr_preload_uninit, NULL); + +void +idr_preload(gfp_t gfp_mask) +{ + struct linux_idr_cache *lic; + struct idr_layer *cacheval; + + sched_pin(); + + lic = &DPCPU_GET(linux_idr_cache); + + /* fill up cache */ + spin_lock(&lic->lock); + while (lic->count < MAX_IDR_FREE) { + spin_unlock(&lic->lock); + cacheval = malloc(sizeof(*cacheval), M_IDR, M_ZERO | gfp_mask); + spin_lock(&lic->lock); + if (cacheval == NULL) + break; + cacheval->ary[0] = lic->head; + lic->head = cacheval; + lic->count++; + } +} + +void +idr_preload_end(void) +{ + struct linux_idr_cache *lic; + + lic = &DPCPU_GET(linux_idr_cache); + spin_unlock(&lic->lock); + sched_unpin(); +} + static inline int idr_max(struct idr *idr) { @@ -280,20 +381,32 @@ idr_pre_get(struct idr *idr, gfp_t gfp_mask) return (1); } -static inline struct idr_layer * -idr_get(struct idr *idr) +static struct idr_layer * +idr_free_list_get(struct idr *idp) { struct idr_layer *il; - il = idr->free; - if (il) { - idr->free = il->ary[0]; + if ((il = idp->free) != NULL) { + idp->free = il->ary[0]; il->ary[0] = NULL; - return (il); } - il = malloc(sizeof(*il), M_IDR, M_ZERO | M_NOWAIT); - if (il != NULL) + return (il); +} + +static inline struct idr_layer * +idr_get(struct idr *idp) +{ + struct idr_layer *il; + + if ((il = idr_free_list_get(idp)) != NULL) { + MPASS(il->bitmap != 0); + } else if ((il = malloc(sizeof(*il), M_IDR, M_ZERO | M_NOWAIT)) != NULL) { bitmap_fill(&il->bitmap, IDR_SIZE); + } else if ((il = idr_preload_dequeue_locked(&DPCPU_GET(linux_idr_cache))) != NULL) { + bitmap_fill(&il->bitmap, IDR_SIZE); + } else { + return (NULL); + } return (il); }