Date: Thu, 15 Jul 2010 18:31:28 +0300 From: Kostik Belousov <kostikbel@gmail.com> To: freebsd-hackers@freebsd.org Subject: Re: Kernel linker and undefined references in KLD Message-ID: <20100715153128.GW2381@deviant.kiev.zoral.com.ua> In-Reply-To: <20100715143235.GU2381@deviant.kiev.zoral.com.ua> References: <AANLkTikhrbdbBEFl-I97nBzdvZx6qBaafDUsveRmYyp3@mail.gmail.com> <20100715143235.GU2381@deviant.kiev.zoral.com.ua>
next in thread | previous in thread | raw e-mail | index | archive | help
--ItroYk2LVxvwOvi/
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Content-Transfer-Encoding: quoted-printable
On Thu, Jul 15, 2010 at 05:32:35PM +0300, Kostik Belousov wrote:
> On Thu, Jul 15, 2010 at 06:07:36PM +0400, Dmitry Krivenok wrote:
> > Hello Hackers,
> >=20
> > I have a question about kernel linker.
> > Please take a look at an example of simple module:
> >=20
> > #######################################################################=
#####
> > #include <sys/param.h>
> > #include <sys/module.h>
> > #include <sys/kernel.h>
> > #include <sys/systm.h>
> > #include <sys/proc.h>
> > #include <sys/sched.h>
> > #include <sys/pcpu.h>
> >=20
> > /* DECLARING A FUNCTION JUST TO AVOID WARNING WHICH IS TREAT AS ERROR
> > BY DEFAULT */
> > void seltdinit(struct thread* t);
> >=20
> > static int event_handler(struct module *module, int event, void *arg)
> > {
> > int e =3D 0;
> > switch (event)
> > {
> > case MOD_LOAD:
> > /* CALLING A FUNCTION DECLARED AS STATIC IN kern/sys_generic.c =
*/
> > seltdinit(curthread);
> > break;
> > case MOD_QUIESCE:
> > break;
> > case MOD_UNLOAD:
> > break;
> > case MOD_SHUTDOWN:
> > break;
> > default:
> > e =3D EOPNOTSUPP;
> > break;
> > }
> >=20
> > return(e);
> > };
> > #######################################################################=
#####
> >=20
> > As you can see, this module calls seltdinit function declared as
> > _static_ in kern/sys_generic.c file.
> > I added a declaration of seltdinit function w/o static just to avoid
> > compilation error since -Werror is used
> > by default in FreeBSD.
> >=20
> > Then I successfully built the module:
> >=20
> > $ make
> > Warning: Object directory not changed from original
> > /usr/home/krived/work/freebsd/trouble
> > cc -O2 -pipe -fno-strict-aliasing -Werror -D_KERNEL -DKLD_MODULE
> > -nostdinc -I. -I@ -I@/contrib/altq -finline-limit=3D8000 --param
> > inline-unit-growth=3D100 --param large-function-growth=3D1000 -fno-comm=
on
> > -fno-omit-frame-pointer -mcmodel=3Dkernel -mno-red-zone -mfpmath=3D387
> > -mno-sse -mno-sse2 -mno-sse3 -mno-mmx -mno-3dnow -msoft-float
> > -fno-asynchronous-unwind-tables -ffreestanding -fstack-protector
> > -std=3Diso9899:1999 -fstack-protector -Wall -Wredundant-decls
> > -Wnested-externs -Wstrict-prototypes -Wmissing-prototypes
> > -Wpointer-arith -Winline -Wcast-qual -Wundef -Wno-pointer-sign
> > -fformat-extensions -c trouble.c
> > ld -d -warn-common -r -d -o trouble.ko trouble.o
> > :> export_syms
> > awk -f /sys/conf/kmod_syms.awk trouble.ko export_syms | xargs -J%
> > objcopy % trouble.ko
> > objcopy --strip-debug trouble.ko
> > $
> >=20
> > As expected, seltdinit symbol is undefined in trouble.ko file:
> > $ nm trouble.ko | grep seltdinit
> > U seltdinit
> > $
> > and is local in kernel binary file:
> > $ nm /boot/kernel/kernel | grep seltdinit
> > ffffffff805fda50 t seltdinit
> > $
> >=20
> > I expected to get something like "undefined reference to seltdinit"
> > error, but the module was loaded w/o any errors:
> >=20
> > $ sudo make load
> > /sbin/kldload -v /usr/home/krived/work/freebsd/trouble/trouble.ko
> > Loaded /usr/home/krived/work/freebsd/trouble/trouble.ko, id=3D2
> > $ kldstat | grep trouble
> > 2 1 0xffffffff81212000 126 trouble.ko
> > $ sudo make unload
> > /sbin/kldunload -v trouble.ko
> > Unloading trouble.ko, id=3D2
> > $
> >=20
> > Could you please explain why the linker (or whatever is loading
> > modules in FreeBSD) doesn't complain?
> >=20
> > For example, ld tool complains for the similar user-space example:
> >=20
> > #######################################################################=
#####
> > $ cat seltdinit.c
> > static void seltdinit(void* td)
> > {
> > return;
> > }
> > $ cat main.c
> > void seltdinit(void* td);
> >=20
> > int main()
> > {
> > seltdinit(0);
> > return 0;
> > }
> > $ gcc -Wall -c seltdinit.c
> > seltdinit.c:2: warning: 'seltdinit' defined but not used
> > $ gcc -Wall -c main.c
> > $ gcc seltdinit.o main.o -o sel
> > main.o: In function `main':
> > main.c:(.text+0xa): undefined reference to `seltdinit'
> > collect2: ld returned 1 exit status
> > $
> > #######################################################################=
#####
> >=20
> > Thanks in advance!
> The kernel linker ignores weak attribute of the symbol, as you see.
> There is more bugs in this department, in regard of the list of
> exported symbols from the modules.
>=20
> I have a patch that fixes the issues, but I am leery to commit it, since
> the fix effectively breaks significant set of the modules.
Ok, below is the patch. I updated it to the latest HEAD, but did not
verified even the build.
diff --git a/sys/conf/kmod_syms.awk b/sys/conf/kmod_syms.awk
index 677d813..98940c6 100644
--- a/sys/conf/kmod_syms.awk
+++ b/sys/conf/kmod_syms.awk
@@ -12,7 +12,10 @@ BEGIN {
=20
# De-list symbols from the export list.
{
- delete syms[$0]
+ split($0, exports)
+ for (member in exports) {
+ delete syms[$member]
+ }
}
=20
# Strip commons, make everything else local.
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index dd29302..f3da962 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -855,7 +855,7 @@ linker_debug_lookup(const char *symstr, c_linker_sym_t =
*sym)
linker_file_t lf;
=20
TAILQ_FOREACH(lf, &linker_files, link) {
- if (LINKER_LOOKUP_SYMBOL(lf, symstr, sym) =3D=3D 0)
+ if (LINKER_LOOKUP_DEBUG_SYMBOL(lf, symstr, sym) =3D=3D 0)
return (0);
}
return (ENOENT);
@@ -899,7 +899,7 @@ linker_debug_symbol_values(c_linker_sym_t sym, linker_s=
ymval_t *symval)
linker_file_t lf;
=20
TAILQ_FOREACH(lf, &linker_files, link) {
- if (LINKER_SYMBOL_VALUES(lf, sym, symval) =3D=3D 0)
+ if (LINKER_DEBUG_SYMBOL_VALUES(lf, sym, symval) =3D=3D 0)
return (0);
}
return (ENOENT);
diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c
index b389ace..f494fa5 100644
--- a/sys/kern/link_elf.c
+++ b/sys/kern/link_elf.c
@@ -127,25 +127,28 @@ typedef struct elf_file {
=20
static int link_elf_link_common_finish(linker_file_t);
static int link_elf_link_preload(linker_class_t cls,
- const char*, linker_file_t*);
+ const char*, linker_file_t*);
static int link_elf_link_preload_finish(linker_file_t);
static int link_elf_load_file(linker_class_t, const char*, linker_file_t*);
static int link_elf_lookup_symbol(linker_file_t, const char*,
- c_linker_sym_t*);
-static int link_elf_symbol_values(linker_file_t, c_linker_sym_t, linker_sy=
mval_t*);
+ c_linker_sym_t*);
+static int link_elf_lookup_debug_symbol(linker_file_t, const char*,
+ c_linker_sym_t*);
+static int link_elf_symbol_values(linker_file_t, c_linker_sym_t,
+ linker_symval_t*);
+static int link_elf_debug_symbol_values(linker_file_t, c_linker_sym_t,
+ linker_symval_t*);
static int link_elf_search_symbol(linker_file_t, caddr_t value,
- c_linker_sym_t* sym, long* diffp);
+ c_linker_sym_t* sym, long* diffp);
=20
static void link_elf_unload_file(linker_file_t);
static void link_elf_unload_preload(linker_file_t);
static int link_elf_lookup_set(linker_file_t, const char *,
void ***, void ***, int *);
static int link_elf_each_function_name(linker_file_t,
- int (*)(const char *, void *),
- void *);
+ int (*)(const char *, void *), void *);
static int link_elf_each_function_nameval(linker_file_t,
- linker_function_nameval_callback_t,
- void *);
+ linker_function_nameval_callback_t, void *);
static void link_elf_reloc_local(linker_file_t);
static long link_elf_symtab_get(linker_file_t, const Elf_Sym **);
static long link_elf_strtab_get(linker_file_t, caddr_t *);
@@ -153,7 +156,9 @@ static Elf_Addr elf_lookup(linker_file_t lf, Elf_Size s=
ymidx, int deps);
=20
static kobj_method_t link_elf_methods[] =3D {
KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol),
+ KOBJMETHOD(linker_lookup_debug_symbol, link_elf_lookup_debug_symbol),
KOBJMETHOD(linker_symbol_values, link_elf_symbol_values),
+ KOBJMETHOD(linker_debug_symbol_values, link_elf_debug_symbol_values),
KOBJMETHOD(linker_search_symbol, link_elf_search_symbol),
KOBJMETHOD(linker_unload, link_elf_unload_file),
KOBJMETHOD(linker_load_file, link_elf_load_file),
@@ -1160,14 +1165,14 @@ elf_hash(const char *name)
}
=20
static int
-link_elf_lookup_symbol(linker_file_t lf, const char* name, c_linker_sym_t*=
sym)
+link_elf_lookup_symbol1(linker_file_t lf, const char* name, c_linker_sym_t=
* sym,
+ int see_local)
{
elf_file_t ef =3D (elf_file_t) lf;
unsigned long symnum;
const Elf_Sym* symp;
const char *strp;
unsigned long hash;
- int i;
=20
/* If we don't have a hash, bail. */
if (ef->buckets =3D=3D NULL || ef->nbuckets =3D=3D 0) {
@@ -1194,60 +1199,101 @@ link_elf_lookup_symbol(linker_file_t lf, const cha=
r* name, c_linker_sym_t* sym)
strp =3D ef->strtab + symp->st_name;
=20
if (strcmp(name, strp) =3D=3D 0) {
- if (symp->st_shndx !=3D SHN_UNDEF ||
- (symp->st_value !=3D 0 &&
- ELF_ST_TYPE(symp->st_info) =3D=3D STT_FUNC)) {
- *sym =3D (c_linker_sym_t) symp;
- return 0;
- } else
- return ENOENT;
+ if (symp->st_shndx !=3D SHN_UNDEF ||
+ (symp->st_value !=3D 0 &&
+ ELF_ST_TYPE(symp->st_info) =3D=3D STT_FUNC)) {
+ if (see_local || ELF_ST_BIND(symp->st_info) !=3D STB_LOCAL) {
+ *sym =3D (c_linker_sym_t) symp;
+ return (0);
+ } else
+ return (ENOENT);
+ } else
+ return (ENOENT);
}
=20
symnum =3D ef->chains[symnum];
}
=20
- /* If we have not found it, look at the full table (if loaded) */
- if (ef->symtab =3D=3D ef->ddbsymtab)
- return ENOENT;
+ return (ENOENT);
+}
=20
- /* Exhaustive search */
- for (i =3D 0, symp =3D ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
- strp =3D ef->ddbstrtab + symp->st_name;
- if (strcmp(name, strp) =3D=3D 0) {
- if (symp->st_shndx !=3D SHN_UNDEF ||
- (symp->st_value !=3D 0 &&
- ELF_ST_TYPE(symp->st_info) =3D=3D STT_FUNC)) {
- *sym =3D (c_linker_sym_t) symp;
- return 0;
- } else
- return ENOENT;
+static int
+link_elf_lookup_symbol(linker_file_t lf, const char* name, c_linker_sym_t*=
sym)
+{
+
+ return (link_elf_lookup_symbol1(lf, name, sym, 0));
+}
+
+static int
+link_elf_lookup_debug_symbol(linker_file_t lf, const char *name,
+ c_linker_sym_t *sym)
+{
+ elf_file_t ef =3D (elf_file_t) lf;
+ const Elf_Sym* symp;
+ const char *strp;
+ int i;
+
+ if (link_elf_lookup_symbol1(lf, name, sym, 1) =3D=3D 0)
+ return (0);
+
+ for (i =3D 0, symp =3D ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
+ strp =3D ef->ddbstrtab + symp->st_name;
+ if (strcmp(name, strp) =3D=3D 0) {
+ if (symp->st_shndx !=3D SHN_UNDEF ||
+ (symp->st_value !=3D 0 &&
+ ELF_ST_TYPE(symp->st_info) =3D=3D STT_FUNC)) {
+ *sym =3D (c_linker_sym_t) symp;
+ return (0);
+ } else
+ return (ENOENT);
+ }
}
- }
=20
- return ENOENT;
+ return (ENOENT);
}
=20
static int
-link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, linker_symval=
_t* symval)
+link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym,
+ linker_symval_t *symval, int see_local)
{
elf_file_t ef =3D (elf_file_t) lf;
const Elf_Sym* es =3D (const Elf_Sym*) sym;
=20
if (es >=3D ef->symtab && es < (ef->symtab + ef->nchains)) {
- symval->name =3D ef->strtab + es->st_name;
- symval->value =3D (caddr_t) ef->address + es->st_value;
- symval->size =3D es->st_size;
- return 0;
+ if (!see_local && ELF_ST_BIND(es->st_info) =3D=3D STB_LOCAL)
+ return (ENOENT);
+ symval->name =3D ef->strtab + es->st_name;
+ symval->value =3D (caddr_t) ef->address + es->st_value;
+ symval->size =3D es->st_size;
+ return (0);
}
- if (ef->symtab =3D=3D ef->ddbsymtab)
- return ENOENT;
+ return (ENOENT);
+}
+
+static int
+link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
+ linker_symval_t *symval)
+{
+
+ return (link_elf_symbol_values1(lf, sym, symval, 0));
+}
+
+static int
+link_elf_debug_symbol_values(linker_file_t lf, c_linker_sym_t sym,
+ linker_symval_t* symval)
+{
+ elf_file_t ef =3D (elf_file_t) lf;
+ const Elf_Sym* es =3D (const Elf_Sym*) sym;
+
+ if (link_elf_symbol_values1(lf, sym, symval, 1) =3D=3D 0)
+ return (0);
if (es >=3D ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
- symval->name =3D ef->ddbstrtab + es->st_name;
- symval->value =3D (caddr_t) ef->address + es->st_value;
- symval->size =3D es->st_size;
- return 0;
+ symval->name =3D ef->ddbstrtab + es->st_name;
+ symval->value =3D (caddr_t) ef->address + es->st_value;
+ symval->size =3D es->st_size;
+ return (0);
}
- return ENOENT;
+ return (ENOENT);
}
=20
static int
diff --git a/sys/kern/link_elf_obj.c b/sys/kern/link_elf_obj.c
index a337fd0..ae8ea04 100644
--- a/sys/kern/link_elf_obj.c
+++ b/sys/kern/link_elf_obj.c
@@ -127,8 +127,12 @@ static int link_elf_link_preload_finish(linker_file_t);
static int link_elf_load_file(linker_class_t, const char *, linker_file_t =
*);
static int link_elf_lookup_symbol(linker_file_t, const char *,
c_linker_sym_t *);
+static int link_elf_lookup_debug_symbol(linker_file_t, const char *,
+ c_linker_sym_t *);
static int link_elf_symbol_values(linker_file_t, c_linker_sym_t,
linker_symval_t *);
+static int link_elf_debug_symbol_values(linker_file_t, c_linker_sym_t,
+ linker_symval_t *);
static int link_elf_search_symbol(linker_file_t, caddr_t value,
c_linker_sym_t *sym, long *diffp);
=20
@@ -148,7 +152,9 @@ static Elf_Addr elf_obj_lookup(linker_file_t lf, Elf_Si=
ze symidx, int deps);
=20
static kobj_method_t link_elf_methods[] =3D {
KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol),
+ KOBJMETHOD(linker_lookup_debug_symbol, link_elf_lookup_debug_symbol),
KOBJMETHOD(linker_symbol_values, link_elf_symbol_values),
+ KOBJMETHOD(linker_debug_symbol_values, link_elf_debug_symbol_values),
KOBJMETHOD(linker_search_symbol, link_elf_search_symbol),
KOBJMETHOD(linker_unload, link_elf_unload_file),
KOBJMETHOD(linker_load_file, link_elf_load_file),
@@ -1067,7 +1073,8 @@ relocate_file(elf_file_t ef)
}
=20
static int
-link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t =
*sym)
+link_elf_lookup_symbol1(linker_file_t lf, const char *name, c_linker_sym_t=
*sym,
+ int see_local)
{
elf_file_t ef =3D (elf_file_t) lf;
const Elf_Sym *symp;
@@ -1077,27 +1084,64 @@ link_elf_lookup_symbol(linker_file_t lf, const char=
*name, c_linker_sym_t *sym)
for (i =3D 0, symp =3D ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
strp =3D ef->ddbstrtab + symp->st_name;
if (symp->st_shndx !=3D SHN_UNDEF && strcmp(name, strp) =3D=3D 0) {
- *sym =3D (c_linker_sym_t) symp;
- return 0;
+ if (see_local ||
+ ELF_ST_BIND(symp->st_info) =3D=3D STB_GLOBAL) {
+ *sym =3D (c_linker_sym_t) symp;
+ return (0);
+ } else
+ return (ENOENT);
}
}
- return ENOENT;
+ return (ENOENT);
}
=20
static int
-link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
- linker_symval_t *symval)
+link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t =
*sym)
+{
+
+ return (link_elf_lookup_symbol1(lf, name, sym, 0));
+}
+
+static int
+link_elf_lookup_debug_symbol(linker_file_t lf, const char *name,
+ c_linker_sym_t *sym)
+{
+
+ return (link_elf_lookup_symbol1(lf, name, sym, 1));
+}
+
+static int
+link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym,
+ linker_symval_t *symval, int see_local)
{
elf_file_t ef =3D (elf_file_t) lf;
const Elf_Sym *es =3D (const Elf_Sym*) sym;
=20
if (es >=3D ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
+ if (!see_local && ELF_ST_BIND(es->st_info) =3D=3D STB_LOCAL)
+ return (ENOENT);
symval->name =3D ef->ddbstrtab + es->st_name;
symval->value =3D (caddr_t)es->st_value;
symval->size =3D es->st_size;
- return 0;
+ return (0);
}
- return ENOENT;
+ return (ENOENT);
+}
+
+static int
+link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
+ linker_symval_t *symval)
+{
+
+ return (link_elf_symbol_values1(lf, sym, symval, 0));
+}
+
+static int
+link_elf_debug_symbol_values(linker_file_t lf, c_linker_sym_t sym,
+ linker_symval_t *symval)
+{
+
+ return (link_elf_symbol_values1(lf, sym, symval, 1));
}
=20
static int
diff --git a/sys/kern/linker_if.m b/sys/kern/linker_if.m
index 3df592c..030e8eb 100644
--- a/sys/kern/linker_if.m
+++ b/sys/kern/linker_if.m
@@ -40,12 +40,24 @@ METHOD int lookup_symbol {
c_linker_sym_t* symp;
};
=20
+METHOD int lookup_debug_symbol {
+ linker_file_t file;
+ const char* name;
+ c_linker_sym_t* symp;
+};
+
METHOD int symbol_values {
linker_file_t file;
c_linker_sym_t sym;
linker_symval_t* valp;
};
=20
+METHOD int debug_symbol_values {
+ linker_file_t file;
+ c_linker_sym_t sym;
+ linker_symval_t* valp;
+};
+
METHOD int search_symbol {
linker_file_t file;
caddr_t value;
@@ -63,6 +75,12 @@ METHOD int each_function_name {
void* opaque;
};
=20
+METHOD int each_debug_function_name {
+ linker_file_t file;
+ linker_function_name_callback_t callback;
+ void* opaque;
+};
+
#
# Call the callback with each specified function and it's value
# defined in the file.
@@ -74,6 +92,12 @@ METHOD int each_function_nameval {
void* opaque;
};
=20
+METHOD int each_debug_function_nameval {
+ linker_file_t file;
+ linker_function_nameval_callback_t callback;
+ void* opaque;
+};
+
#
# Search for a linker set in a file. Return a pointer to the first
# entry (which is itself a pointer), and the number of entries.
diff --git a/sys/modules/ata/atacore/Makefile b/sys/modules/ata/atacore/Mak=
efile
index 7b8d9e2..30a4ccb 100644
--- a/sys/modules/ata/atacore/Makefile
+++ b/sys/modules/ata/atacore/Makefile
@@ -6,4 +6,6 @@ KMOD=3D ata
SRCS=3D ata-all.c ata-lowlevel.c ata-queue.c ata_if.c
SRCS+=3D opt_ata.h ata_if.h device_if.h bus_if.h
=20
+EXPORT_SYMS=3D YES
+
.include <bsd.kmod.mk>
diff --git a/sys/modules/mii/Makefile b/sys/modules/mii/Makefile
index 6232b5e..9740cf2 100644
--- a/sys/modules/mii/Makefile
+++ b/sys/modules/mii/Makefile
@@ -12,7 +12,11 @@ SRCS+=3D rgephy.c rlphy.c ruephy.c tdkphy.c tlphy.c true=
phy.c ukphy.c
SRCS+=3D ukphy_subr.c
SRCS+=3D xmphy.c
=20
-EXPORT_SYMS=3D mii_mediachg \
+EXPORT_SYMS=3D miibus_devclass \
+ miibus_driver \
+ miibus_readreg_desc \
+ miibus_writereg_desc \
+ mii_mediachg \
mii_phy_probe \
mii_phy_reset \
mii_pollstat \
diff --git a/sys/modules/pseudofs/Makefile b/sys/modules/pseudofs/Makefile
index d5696c5..ea7b112 100644
--- a/sys/modules/pseudofs/Makefile
+++ b/sys/modules/pseudofs/Makefile
@@ -11,6 +11,7 @@ SRCS=3D opt_pseudofs.h \
pseudofs_vnops.c
=20
EXPORT_SYMS=3D pfs_mount \
+ pfs_cmount \
pfs_unmount \
pfs_root \
pfs_statfs \
--ItroYk2LVxvwOvi/
Content-Type: application/pgp-signature
Content-Disposition: inline
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.10 (FreeBSD)
iEYEARECAAYFAkw/Kc8ACgkQC3+MBN1Mb4iSpACbBe0l9Xy0cbXDidyeeaoEF+Q1
rfgAnjUtFmHLJc893selq/jjF6OJsyfq
=3Gwr
-----END PGP SIGNATURE-----
--ItroYk2LVxvwOvi/--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20100715153128.GW2381>