Date: Fri, 1 Aug 2014 13:33:47 +0000 (UTC) From: Gleb Smirnoff <glebius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r269377 - in projects/lwref/sys: amd64/amd64 conf kern sys Message-ID: <201408011333.s71DXldw096650@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: glebius Date: Fri Aug 1 13:33:47 2014 New Revision: 269377 URL: http://svnweb.freebsd.org/changeset/base/269377 Log: Drop into subversion the proof-of-concept unfinished and dirty implementation of light weight reference counting mechanism. Added: projects/lwref/sys/amd64/amd64/lwref.S (contents, props changed) projects/lwref/sys/kern/subr_lwref.c (contents, props changed) projects/lwref/sys/sys/lwref.h (contents, props changed) Modified: projects/lwref/sys/amd64/amd64/apic_vector.S projects/lwref/sys/conf/files projects/lwref/sys/conf/files.amd64 projects/lwref/sys/kern/sched_ule.c projects/lwref/sys/sys/sched.h Modified: projects/lwref/sys/amd64/amd64/apic_vector.S ============================================================================== --- projects/lwref/sys/amd64/amd64/apic_vector.S Fri Aug 1 12:42:37 2014 (r269376) +++ projects/lwref/sys/amd64/amd64/apic_vector.S Fri Aug 1 13:33:47 2014 (r269377) @@ -70,6 +70,8 @@ IDTVEC(vec_name) ; \ movq %rsp, %rsi ; \ movl %eax, %edi ; /* pass the IRQ */ \ call lapic_handle_intr ; \ +.globl vec_name ## _ret ; \ +vec_name ## _ret : ; \ 1: ; \ MEXITCOUNT ; \ jmp doreti @@ -107,6 +109,8 @@ IDTVEC(timerint) FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp, %rdi call lapic_handle_timer +.globl timerint_ret +timerint_ret: MEXITCOUNT jmp doreti @@ -234,6 +238,8 @@ IDTVEC(ipi_intr_bitmap_handler) FAKE_MCOUNT(TF_RIP(%rsp)) call ipi_bitmap_handler +.globl ipi_intr_bitmap_handler_ret +ipi_intr_bitmap_handler_ret: MEXITCOUNT jmp doreti Added: projects/lwref/sys/amd64/amd64/lwref.S ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/lwref/sys/amd64/amd64/lwref.S Fri Aug 1 13:33:47 2014 (r269377) @@ -0,0 +1,32 @@ +#include <machine/asmacros.h> + +/* + * void *lwref_acquire(lwref_t lwr, counter_u64_t *cp) + * { + * void *ptr; + * + * ptr = lwr->ptr; + * cp = &lwr->refs; + * + * counter_u64_add(*cp, 1); + * + * return (ptr); + * } + */ + +ENTRY(lwref_acquire) + mov (%rdi), %rax + mov 0x8(%rdi), %rcx + mov %rcx, (%rsi) + mov $__pcpu, %rdx + sub %rdx, %rcx + mov $100000, %r10 +cycle: + sub $1, %r10 + cmpq $0, %r10 + jne cycle + addq $1, %gs:(%rcx) +.globl lwref_acquire_ponr +lwref_acquire_ponr: + ret +END(lwref_acquire) Modified: projects/lwref/sys/conf/files ============================================================================== --- projects/lwref/sys/conf/files Fri Aug 1 12:42:37 2014 (r269376) +++ projects/lwref/sys/conf/files Fri Aug 1 13:33:47 2014 (r269377) @@ -2972,6 +2972,7 @@ kern/subr_kdb.c standard kern/subr_kobj.c standard kern/subr_lock.c standard kern/subr_log.c standard +kern/subr_lwref.c standard kern/subr_mbpool.c optional libmbpool kern/subr_mchain.c optional libmchain kern/subr_module.c standard Modified: projects/lwref/sys/conf/files.amd64 ============================================================================== --- projects/lwref/sys/conf/files.amd64 Fri Aug 1 12:42:37 2014 (r269376) +++ projects/lwref/sys/conf/files.amd64 Fri Aug 1 13:33:47 2014 (r269377) @@ -109,6 +109,7 @@ amd64/amd64/initcpu.c standard amd64/amd64/io.c optional io amd64/amd64/locore.S standard no-obj amd64/amd64/xen-locore.S optional xenhvm +amd64/amd64/lwref.S standard amd64/amd64/machdep.c standard amd64/amd64/mem.c optional mem amd64/amd64/minidump_machdep.c standard Modified: projects/lwref/sys/kern/sched_ule.c ============================================================================== --- projects/lwref/sys/kern/sched_ule.c Fri Aug 1 12:42:37 2014 (r269376) +++ projects/lwref/sys/kern/sched_ule.c Fri Aug 1 13:33:47 2014 (r269377) @@ -2707,6 +2707,27 @@ sched_fork_exit(struct thread *td) } /* + * Apply a function to every thread on runqueue. + */ +void +sched_foreach_on_runq(void(*func)(void *)) +{ + struct tdq *tdq; + struct thread *td; + + tdq = TDQ_SELF(); + + for (int i = 0; i < RQ_NQS; i++) { + TAILQ_FOREACH(td, &tdq->tdq_realtime.rq_queues[i], td_runq) + (func)(td); + TAILQ_FOREACH(td, &tdq->tdq_timeshare.rq_queues[i], td_runq) + (func)(td); + TAILQ_FOREACH(td, &tdq->tdq_idle.rq_queues[i], td_runq) + (func)(td); + } +} + +/* * Create on first use to catch odd startup conditons. */ char * Added: projects/lwref/sys/kern/subr_lwref.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/lwref/sys/kern/subr_lwref.c Fri Aug 1 13:33:47 2014 (r269377) @@ -0,0 +1,208 @@ +/*- + * Copyright (c) 2014 Gleb Smirnoff <glebius@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/lock.h> +#include <sys/malloc.h> /* XXXGL: M_TEMP */ +#include <sys/mutex.h> +#include <sys/lwref.h> +#include <sys/proc.h> +#include <sys/sched.h> +#include <sys/smp.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/pcb.h> + +struct lwref { + void *ptr; + counter_u64_t refcnt; + struct mtx mtx; +}; + +static void lwref_change_action(void *v); + +#ifdef INVARIANTS +extern void Xrendezvous(void); +extern void Xtimerint(void); +#endif + +lwref_t +lwref_alloc(void *ptr, int flags) +{ + lwref_t lwr; + + lwr = malloc(sizeof(*lwr), M_TEMP, flags | M_ZERO); + if (lwr == NULL) + return (NULL); + lwr->refcnt = counter_u64_alloc(flags); + if (lwr->refcnt == NULL) { + free(lwr, M_TEMP); + return (NULL); + } + lwr->ptr = ptr; + mtx_init(&lwr->mtx, "lwref", NULL, MTX_DEF); + + return (lwr); +} + +struct lwref_change_ctx { + lwref_t lwr; + void *newptr; + counter_u64_t newcnt; + u_int cpu; + u_int oldcnt; +}; + +static void +lwref_fixup_rip(register_t *rip, const char *p) +{ + + if (*rip >= (register_t )lwref_acquire && + *rip < (register_t )lwref_acquire_ponr) { + if (p) + printf("%s: %p\n", p, (void *)*rip); + *rip = (register_t )lwref_acquire; + } +} + +static void +lwref_fixup_td(void *arg) +{ + struct thread *td = arg; + register_t *rbp; + + /* + * The timer interrupt trapframe is 3 functions deep: + * Xtimerint -> lapic_handle_timer -> mi_switch -> sched_switch, + * so in 99% this would work: + * + * tf = (struct trapframe *) + * ((register_t *)(***(void ****)(td->td_pcb->pcb_rbp)) + 2); + * + */ + for (rbp = (register_t *)td->td_pcb->pcb_rbp; + rbp && rbp < (register_t *)*rbp; + rbp = (register_t *)*rbp) { + struct trapframe *tf; + register_t rip = (register_t )*(rbp + 1); + + if ( + rip == (register_t )timerint_ret || + rip == (register_t )apic_isr1_ret || + rip == (register_t )apic_isr2_ret || + rip == (register_t )apic_isr3_ret || + rip == (register_t )apic_isr4_ret || + rip == (register_t )apic_isr5_ret || + rip == (register_t )apic_isr6_ret || + rip == (register_t )apic_isr7_ret || + rip == (register_t )ipi_intr_bitmap_handler_ret + ) { + struct trapframe *tf; + + tf = (struct trapframe *)(rbp + 2); + lwref_fixup_rip(&tf->tf_rip, __func__); + } + + tf = (struct trapframe *)(rbp + 2); + if (tf->tf_rip > (register_t )lwref_acquire && + tf->tf_rip < (register_t )lwref_acquire_ponr) + panic("lwref deteceted\n"); + } +} + +static void +lwref_change_action(void *v) +{ + struct lwref_change_ctx *ctx = v; + lwref_t lwr = ctx->lwr; + struct trapframe *tf; + + atomic_add_int(&ctx->oldcnt, *(uint64_t *)zpcpu_get(lwr->refcnt)); + + lwr->ptr = ctx->newptr; + lwr->refcnt = ctx->newcnt; + + sched_foreach_on_runq(lwref_fixup_td); + + if (ctx->cpu == curcpu) + /* We are not in IPI. */ + return; + + /* + * We are in IPI and we need to check the trap frame of + * the IPI, whether we interrupted lwref_acquire(). + * + * We are two functions deep below the trap frame: + * Xrendezvous -> smp_rendezvous_action -> lwref_change_action + */ + KASSERT(__builtin_return_address(0) > (void *)&smp_rendezvous_action && + __builtin_return_address(0) <= (void *)((char *)&smp_rendezvous_action + 506) && + __builtin_return_address(1) > (void *)&Xrendezvous && + __builtin_return_address(1) <= (void *)((char *)&Xrendezvous + 201), + ("%p called via invalid path: 0 %p 1 %p", __func__, + __builtin_return_address(0), __builtin_return_address(1))); + + /* After pushed %rbp and %rip begins the trap frame. */ + tf = (struct trapframe *) + ((register_t *)__builtin_frame_address(1) + 2); + lwref_fixup_rip(&tf->tf_rip, __func__); +} + +int +lwref_change(lwref_t lwr, void *newptr, void (*freefn)(void *, void *), + void *freearg) +{ + struct lwref_change_ctx ctx; + counter_u64_t orefcnt; + void *optr; + + ctx.newcnt = counter_u64_alloc(M_WAITOK); /* XXXGL */ + ctx.oldcnt = 0; + + mtx_lock(&lwr->mtx); + optr = lwr->ptr; + orefcnt = lwr->refcnt; + ctx.lwr = lwr; + ctx.newptr = newptr; + ctx.cpu = curcpu; /* XXXGL: race */ + smp_rendezvous(smp_no_rendevous_barrier, lwref_change_action, + smp_no_rendevous_barrier, &ctx); + mtx_unlock(&lwr->mtx); + + if (ctx.oldcnt == 0) { + (freefn)(freearg, optr); + counter_u64_free(orefcnt); + } else + printf("Leaking %p with cnt %p %u (%ju) refs\n", + optr, orefcnt, ctx.oldcnt, (uintmax_t )counter_u64_fetch(orefcnt)); + + return (0); +} Added: projects/lwref/sys/sys/lwref.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/lwref/sys/sys/lwref.h Fri Aug 1 13:33:47 2014 (r269377) @@ -0,0 +1,63 @@ +/*- + * Copyright (c) 2014 Gleb Smirnoff <glebius@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __SYS_LWREF_H__ +#define __SYS_LWREF_H__ + +#include <sys/counter.h> + +struct lwref; +typedef struct lwref * lwref_t; + +lwref_t lwref_alloc(void *, int); +int lwref_change(lwref_t, void *, void(*)(void *, void *), void *); + +/* asm */ +void *lwref_acquire(lwref_t, counter_u64_t *); +extern char lwref_acquire_ponr[]; + +extern char timerint_ret[]; +extern char apic_isr1_ret[]; +extern char apic_isr2_ret[]; +extern char apic_isr3_ret[]; +extern char apic_isr4_ret[]; +extern char apic_isr5_ret[]; +extern char apic_isr6_ret[]; +extern char apic_isr7_ret[]; +extern char ipi_intr_bitmap_handler_ret[]; + +#ifdef INVARIANTS +#define lwref_release(p, c) do { \ + p = NULL; \ + counter_u64_add(c, -1); \ +} while (0) +#else +#define lwref_release(p, c) counter_u64_add(c, -1) +#endif + +#endif /* ! __SYS_LWREF_H__ */ Modified: projects/lwref/sys/sys/sched.h ============================================================================== --- projects/lwref/sys/sys/sched.h Fri Aug 1 12:42:37 2014 (r269376) +++ projects/lwref/sys/sys/sched.h Fri Aug 1 13:33:47 2014 (r269377) @@ -147,6 +147,11 @@ char *sched_tdname(struct thread *td); void sched_clear_tdname(struct thread *td); #endif +/* + * Used for lwref fixups. + */ +void sched_foreach_on_runq(void(*)(void *)); + static __inline void sched_pin(void) {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201408011333.s71DXldw096650>