From owner-freebsd-smp  Sat Dec 14 13:56:03 1996
Return-Path: <owner-smp>
Received: (from root@localhost)
          by freefall.freebsd.org (8.8.4/8.8.4) id NAA17809
          for smp-outgoing; Sat, 14 Dec 1996 13:56:03 -0800 (PST)
Received: from clem.systemsix.com (clem.systemsix.com [198.99.86.131])
          by freefall.freebsd.org (8.8.4/8.8.4) with SMTP id NAA17776
          for <smp@freebsd.org>; Sat, 14 Dec 1996 13:55:55 -0800 (PST)
Received: from localhost (localhost [127.0.0.1]) by clem.systemsix.com (8.6.12/8.6.12) with SMTP id OAA17577; Sat, 14 Dec 1996 14:51:50 -0700
Message-Id: <199612142151.OAA17577@clem.systemsix.com>
X-Authentication-Warning: clem.systemsix.com: Host localhost didn't use HELO protocol
X-Mailer: exmh version 1.6.5 12/11/95
From: Steve Passe <smp@csn.net>
To: Steve Passe <smp@csn.net>
cc: haertel@ichips.intel.com, peter@spinner.dialix.com, dg@root.com,
        smp@freebsd.org, toor@dyson.iquest.net
Subject: Re: some questions concerning TLB shootdowns in FreeBSD 
In-reply-to: Your message of "Sat, 14 Dec 1996 11:55:18 MST."
             <199612141855.LAA16724@clem.systemsix.com> 
Mime-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Date: Sat, 14 Dec 1996 14:51:50 -0700
Sender: owner-smp@freebsd.org
X-Loop: FreeBSD.org
Precedence: bulk

Hi,

so here is a suggested set of code for the TLB sync problem.  There is one
problem with it (that I currently see, surely others also!) in that the
target CPUs can't service their IPI as the invoking CPU holds the mp_lock.
So for now lets pretend that we have a separate lock for IPIs called
ipi_lock, which is manipulated via get_ipilock()/rel_ipilock().

---
usage by the invoking CPU:

	startRendezvous();		/* setup a rendezvous */

	/* 
	 * at this point the other CPUs are all spinning on the end lock
	 * so the code can safely muck with PTD/PTE entries...
	 */

	invltlb();			/* CPU flushes local TLB */

	endRendezvous();		/* end the rendezvous */

---
usage by the invoked CPUs, ie the routine invoked by the IPI:

ipi_invltlb(void)
{
    u_long	temp;

    doRendezvous();			/* declare our arrival and wait */

    __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
			 : : "memory");
}

----------------------------------- cut -------------------------------------
/* rendezvous.s */

	.text
	.align	4

#define SMP_INVLTLB_IPI		(ICU_OFFSET+27)

/*
 * invoking CPU sets up rendezvous
 */
ENTRY(startRendezvous)
	call	_get_ipilock			/* only one CPU at a time */

	movl	_mp_ncpus, %eax			/* # of CPUs to sync */
	decl	%eax				/* count ourself */
	movl	%eax, _rendezvousCount		/* init the downcounter */
	movl	%eax, _rendezvousEnd		/* init the release lock */

	pushl	SMP_INVLTLB_IPI
	call	_allButSelfIPI
	addl	$4, %esp

	call	_rel_ipilock			/* now safe for other CPUs */

1:	cmpl	$0, _rendezvousCount		/* check current value */
	jnz	1b				/* somebody not here yet */

	call	_get_ipilock			/* is this necessary??? */
	ret

/*
 * invoking CPU releases all other CPUs
 */
ENTRY(endRendezvous)
	movl	$0, _rendezvousEnd
	call	_rel_ipilock			/* is this necessary??? */
	ret

/*
 * invoked CPUs enter and wait for end
 */
ENTRY(doRendezvous)
	call	_rel_ipilock			/* allow other CPUs to IPI */

	lock					/* ensure atomic operation */
	decl	_rendezvousCount		/* declare our arrival */

1:	cmpl	$0, _rendezvousEnd		/* test for end */
	jnz	1b				/* not yet, spin */

	call	_get_ipilock			/* safe exit from IPI */
	ret

	.data
	ALIGN_DATA

	.globl _rendezvousCount
_rendezvousCount:
	.long 0

	.globl _rendezvousEnd
_rendezvousEnd:
	.long 0

----------------------------------- cut -------------------------------------

--
Steve Passe	| powered by
smp@csn.net	|            FreeBSD