Date: Fri, 29 Jun 2012 10:39:40 GMT From: Alexandr <alter@alter.org.ua> To: freebsd-gnats-submit@FreeBSD.org Subject: kern/169539: running gmirror on MSI MegaRaid (/dev/ar* vs /dev/gm*) Message-ID: <201206291039.q5TAdeTL024312@red.freebsd.org> Resent-Message-ID: <201206291040.q5TAe4Iv075961@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 169539
>Category: kern
>Synopsis: running gmirror on MSI MegaRaid (/dev/ar* vs /dev/gm*)
>Confidential: no
>Severity: non-critical
>Priority: low
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: change-request
>Submitter-Id: current-users
>Arrival-Date: Fri Jun 29 10:40:04 UTC 2012
>Closed-Date:
>Last-Modified:
>Originator: Alexandr
>Release: 7.2 and CURRENT
>Organization:
Alfa-inet ISP
>Environment:
>Description:
LSI MegaRaid controller
* always modifies last sector of attached disks.
* can't boot from disks without its own (LSI) metadata.
* there is no passthrough option to expose disk as non-raid storage.
This makes impossible to boot from GMIRROR array on this controller.
Since ar* handle LSI metadata in r/o mode, I can't maintain array without BIOS.
This in unacceptable.
Using gmirror also becomes impossible.
>How-To-Repeat:
1) config GMIRROR on boot disk
2) reboot
ok, now we have no valid metadata at all (neither gm*, nor ar*)
system cannot boot
>Fix:
Fast solution:
http://alter.org.ua/en/soft/fbsd/gmirror/
(patch for CURRENT is attached)
Update gmirror with special patch. Now it can optionally store its metadata in the previous sector. You can use -o 1 option with 'label' and 'insert' to place gmirror metadata in alternative location.
gmirror attempts to load metadata from last sector. If this attempt fails, it tries previous one. gmirror shall not ovewrite last sector if metadata is stored in previous. So, we have compatibility option for ar*.
Setup BIOS to use all drives as separate STRIPE arrays (one disk in each one).
Use gmirror with -o 1 option to setup proper array.
---
Right solution (imho):
I think, would be nice to add some API or library to report presence and location of some known metadata.
Use this API in all RAID tools (atacontrol, gmirror) to check and warn about presence of incompatible matadata on disks.
And (if required to workaroud some hardware issues) place metadata in alternative locations.
Patch attached with submission follows:
diff -ruN orig/sbin/geom/class/mirror/geom_mirror.c new/sbin/geom/class/mirror/geom_mirror.c
--- orig/sbin/geom/class/mirror/geom_mirror.c 2012-06-15 21:22:38.000000000 +0300
+++ new/sbin/geom/class/mirror/geom_mirror.c 2012-06-18 01:09:23.000000000 +0300
@@ -47,6 +47,7 @@
#define GMIRROR_BALANCE "load"
#define GMIRROR_SLICE "4096"
#define GMIRROR_PRIORITY "0"
+static intmax_t label_offset = 0;
static void mirror_main(struct gctl_req *req, unsigned flags);
static void mirror_activate(struct gctl_req *req);
@@ -72,9 +73,10 @@
{ 'n', "noautosync", NULL, G_TYPE_BOOL },
{ 'p', "priority", "-1", G_TYPE_NUMBER },
{ 's', "slice", "-1", G_TYPE_NUMBER },
+ { 'o', "mdoffset", &label_offset, G_TYPE_NUMBER },
G_OPT_SENTINEL
},
- "[-adfFhnv] [-b balance] [-s slice] name\n"
+ "[-adfFhnv] [-o offset] [-b balance] [-s slice] name\n"
"[-v] -p priority name prov"
},
{ "deactivate", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
@@ -93,6 +95,7 @@
{ 'h', "hardcode", NULL, G_TYPE_BOOL },
{ 'n', "noautosync", NULL, G_TYPE_BOOL },
{ 's', "slice", GMIRROR_SLICE, G_TYPE_NUMBER },
+ { 'o', "mdoffset", &label_offset, G_TYPE_NUMBER },
G_OPT_SENTINEL
},
"[-Fhnv] [-b balance] [-s slice] name prov ..."
@@ -102,9 +105,10 @@
{ 'h', "hardcode", NULL, G_TYPE_BOOL },
{ 'i', "inactive", NULL, G_TYPE_BOOL },
{ 'p', "priority", GMIRROR_PRIORITY, G_TYPE_NUMBER },
+ { 'o', "mdoffset", &label_offset, G_TYPE_NUMBER },
G_OPT_SENTINEL
},
- "[-hiv] [-p priority] name prov ..."
+ "[-hiv] [-o offset] [-p priority] name prov ..."
},
{ "rebuild", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
"[-v] name prov ..."
@@ -165,6 +169,11 @@
gctl_error(req, "Too few arguments.");
return;
}
+ label_offset = gctl_get_intmax(req, "mdoffset");
+ if(label_offset != 0 && label_offset > 1) {
+ gctl_error(req, "Invalid metadata offset %d.", (int)label_offset);
+ return;
+ }
strlcpy(md.md_magic, G_MIRROR_MAGIC, sizeof(md.md_magic));
md.md_version = G_MIRROR_VERSION;
@@ -227,9 +236,9 @@
*/
for (i = 1; i < nargs; i++) {
str = gctl_get_ascii(req, "arg%d", i);
- error = g_metadata_clear(str, NULL);
+ error = g_metadata_clear_ex(str, NULL, label_offset);
if (error != 0) {
- gctl_error(req, "Can't store metadata on %s: %s.", str,
+ gctl_error(req, "Can't clear metadata on %s[%d]: %s.", str, (int)label_offset,
strerror(error));
return;
}
@@ -252,15 +261,15 @@
strlcpy(md.md_provider, str, sizeof(md.md_provider));
}
mirror_metadata_encode(&md, sector);
- error = g_metadata_store(str, sector, sizeof(sector));
+ error = g_metadata_store_ex(str, sector, sizeof(sector), label_offset);
if (error != 0) {
- fprintf(stderr, "Can't store metadata on %s: %s.\n",
- str, strerror(error));
+ fprintf(stderr, "Can't store metadata on %s[%d]: %s.\n",
+ str, (int)label_offset, strerror(error));
gctl_error(req, "Not fully done.");
continue;
}
if (verbose)
- printf("Metadata value stored on %s.\n", str);
+ printf("Metadata value stored on %s[%d].\n", str, (int)label_offset);
}
}
@@ -278,9 +287,12 @@
for (i = 0; i < nargs; i++) {
name = gctl_get_ascii(req, "arg%d", i);
- error = g_metadata_clear(name, G_MIRROR_MAGIC);
+ error = g_metadata_clear_ex(name, G_MIRROR_MAGIC, 0);
+ if (error != 0) {
+ error = g_metadata_clear_ex(name, G_MIRROR_MAGIC, 1);
+ }
if (error != 0) {
- fprintf(stderr, "Can't clear metadata on %s: %s.\n",
+ fprintf(stderr, "Can't clear metadata on %s[0,1]: %s.\n",
name, strerror(error));
gctl_error(req, "Not fully done.");
continue;
@@ -305,8 +317,18 @@
for (i = 0; i < nargs; i++) {
name = gctl_get_ascii(req, "arg%d", i);
- error = g_metadata_read(name, (u_char *)&tmpmd, sizeof(tmpmd),
- G_MIRROR_MAGIC);
+ label_offset = 0;
+ error = g_metadata_read_ex(name, (u_char *)&tmpmd, sizeof(tmpmd),
+ G_MIRROR_MAGIC, 0);
+ if (error != 0) {
+ error = g_metadata_read_ex(name, (u_char *)&tmpmd, sizeof(tmpmd),
+ G_MIRROR_MAGIC, 1);
+ if (error == 0) {
+ printf("Found metadata on %s at offset 1.\n",
+ name);
+ label_offset = 1;
+ }
+ }
if (error != 0) {
fprintf(stderr, "Can't read metadata from %s: %s.\n",
name, strerror(error));
@@ -319,7 +341,7 @@
gctl_error(req, "Not fully done.");
continue;
}
- printf("Metadata on %s:\n", name);
+ printf("Metadata on %s[%d]:\n", name, (int)label_offset);
mirror_metadata_dump(&md);
printf("\n");
}
@@ -341,8 +363,18 @@
for (i = 1; i < nargs; i++) {
path = gctl_get_ascii(req, "arg%d", i);
- error = g_metadata_read(path, (u_char *)&tmpmd, sizeof(tmpmd),
- G_MIRROR_MAGIC);
+ label_offset = 0;
+ error = g_metadata_read_ex(path, (u_char *)&tmpmd, sizeof(tmpmd),
+ G_MIRROR_MAGIC, 0);
+ if (error != 0) {
+ error = g_metadata_read_ex(name, (u_char *)&tmpmd, sizeof(tmpmd),
+ G_MIRROR_MAGIC, 1);
+ if (error == 0) {
+ printf("Found metadata on %s at offset 1.\n",
+ name);
+ label_offset = 1;
+ }
+ }
if (error != 0) {
fprintf(stderr, "Cannot read metadata from %s: %s.\n",
path, strerror(error));
@@ -365,7 +397,7 @@
}
md.md_dflags &= ~G_MIRROR_DISK_FLAG_INACTIVE;
mirror_metadata_encode(&md, (u_char *)&tmpmd);
- error = g_metadata_store(path, (u_char *)&tmpmd, sizeof(tmpmd));
+ error = g_metadata_store_ex(path, (u_char *)&tmpmd, sizeof(tmpmd), label_offset);
if (error != 0) {
fprintf(stderr, "Cannot write metadata from %s: %s.\n",
path, strerror(error));
@@ -373,6 +405,6 @@
continue;
}
if (verbose)
- printf("Provider %s activated.\n", path);
+ printf("Provider %s[%d] activated.\n", path, (int)label_offset);
}
}
diff -ruN orig/sbin/geom/class/mirror/gmirror.8 new/sbin/geom/class/mirror/gmirror.8
--- orig/sbin/geom/class/mirror/gmirror.8 2012-06-15 21:22:38.000000000 +0300
+++ new/sbin/geom/class/mirror/gmirror.8 2012-06-18 01:32:34.000000000 +0300
@@ -36,6 +36,7 @@
.Op Fl Fhnv
.Op Fl b Ar balance
.Op Fl s Ar slice
+.Op Fl o Ar offset
.Ar name
.Ar prov ...
.Nm
@@ -63,6 +64,7 @@
.Cm insert
.Op Fl hiv
.Op Fl p Ar priority
+.Op Fl o Ar offset
.Ar name
.Ar prov ...
.Nm
@@ -158,6 +160,10 @@
the I/O request will be split into N pieces, where N is the number of active
components.
Defaults to 4096 bytes.
+.It Fl o Ar offset
+Specifies metadata offset (0 - standard or 1 - before last sector)..
+.Cm split
+Alternative metadata location is used to workaround Soft-RAIDs, those always overwrite last sector on boot
.El
.It Cm clear
Clear metadata on the given providers.
@@ -188,6 +194,8 @@
Specifies slice size for
.Cm split
balance algorithm.
+.It Fl o Ar offset
+Specifies metadata offset
.El
.It Cm rebuild
Rebuild the given mirror components forcibly.
@@ -204,6 +212,8 @@
Mark component(s) as inactive immediately after insertion.
.It Fl p Ar priority
Specifies priority of the given component(s).
+.It Fl o Ar offset
+Specifies metadata offset.
.El
.It Cm remove
Remove the given component(s) from the mirror and clear metadata on it.
diff -ruN orig/sbin/geom/misc/subr.c new/sbin/geom/misc/subr.c
--- orig/sbin/geom/misc/subr.c 2012-06-15 21:22:39.000000000 +0300
+++ new/sbin/geom/misc/subr.c 2012-06-18 01:15:36.000000000 +0300
@@ -220,8 +220,14 @@
}
int
-g_metadata_read(const char *name, unsigned char *md, size_t size,
- const char *magic)
+g_metadata_read(const char *name, unsigned char *md, size_t size, const char *magic)
+{
+ return g_metadata_read_ex(name, md, size, magic, 0);
+}
+
+int
+g_metadata_read_ex(const char *name, unsigned char *md, size_t size,
+ const char *magic, size_t md_offset)
{
struct std_metadata stdmd;
unsigned char *sector;
@@ -251,7 +257,7 @@
error = ENOMEM;
goto out;
}
- if (pread(fd, sector, sectorsize, mediasize - sectorsize) !=
+ if (pread(fd, sector, sectorsize, mediasize - sectorsize*(1+md_offset)) !=
sectorsize) {
error = errno;
goto out;
@@ -274,6 +280,12 @@
int
g_metadata_store(const char *name, const unsigned char *md, size_t size)
{
+ return g_metadata_store_ex(name, md, size, 0);
+}
+
+int
+g_metadata_store_ex(const char *name, const unsigned char *md, size_t size, size_t md_offset)
+{
unsigned char *sector;
ssize_t sectorsize;
off_t mediasize;
@@ -302,7 +314,7 @@
goto out;
}
bcopy(md, sector, size);
- if (pwrite(fd, sector, sectorsize, mediasize - sectorsize) !=
+ if (pwrite(fd, sector, sectorsize, mediasize - sectorsize*(1+md_offset)) !=
sectorsize) {
error = errno;
goto out;
@@ -318,6 +330,12 @@
int
g_metadata_clear(const char *name, const char *magic)
{
+ return g_metadata_clear_ex(name, magic, 0);
+}
+
+int
+g_metadata_clear_ex(const char *name, const char *magic, size_t md_offset)
+{
struct std_metadata md;
unsigned char *sector;
ssize_t sectorsize;
@@ -346,7 +364,7 @@
goto out;
}
if (magic != NULL) {
- if (pread(fd, sector, sectorsize, mediasize - sectorsize) !=
+ if (pread(fd, sector, sectorsize, mediasize - sectorsize*(1+md_offset)) !=
sectorsize) {
error = errno;
goto out;
@@ -358,7 +376,7 @@
}
}
bzero(sector, sectorsize);
- if (pwrite(fd, sector, sectorsize, mediasize - sectorsize) !=
+ if (pwrite(fd, sector, sectorsize, mediasize - sectorsize*(1+md_offset)) !=
sectorsize) {
error = errno;
goto out;
diff -ruN orig/sbin/geom/misc/subr.h new/sbin/geom/misc/subr.h
--- orig/sbin/geom/misc/subr.h 2012-06-15 21:22:39.000000000 +0300
+++ new/sbin/geom/misc/subr.h 2012-06-18 01:17:08.000000000 +0300
@@ -39,8 +39,12 @@
int g_metadata_read(const char *name, unsigned char *md, size_t size,
const char *magic);
+int g_metadata_read_ex(const char *name, unsigned char *md, size_t size,
+ const char *magic, size_t md_offset);
int g_metadata_store(const char *name, const unsigned char *md, size_t size);
+int g_metadata_store_ex(const char *name, const unsigned char *md, size_t size, size_t md_offset);
int g_metadata_clear(const char *name, const char *magic);
+int g_metadata_clear_ex(const char *name, const char *magic, size_t md_offset);
void gctl_error(struct gctl_req *req, const char *error, ...) __printflike(2, 3);
int gctl_get_int(struct gctl_req *req, const char *pfmt, ...) __printflike(2, 3);
diff -ruN orig/sys/geom/mirror/g_mirror.c new/sys/geom/mirror/g_mirror.c
--- orig/sys/geom/mirror/g_mirror.c 2012-06-16 01:08:42.000000000 +0300
+++ new/sys/geom/mirror/g_mirror.c 2012-06-18 01:22:01.000000000 +0300
@@ -437,7 +437,7 @@
*/
static struct g_mirror_disk *
g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
- struct g_mirror_metadata *md, int *errorp)
+ struct g_mirror_metadata *md, int *errorp, u_int d_md_offset)
{
struct g_mirror_disk *disk;
int error;
@@ -462,6 +462,7 @@
disk->d_sync.ds_offset_done = md->md_sync_offset;
disk->d_genid = md->md_genid;
disk->d_sync.ds_syncid = md->md_syncid;
+ disk->d_md_offset = d_md_offset;
if (errorp != NULL)
*errorp = 0;
return (disk);
@@ -632,7 +633,7 @@
("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
cp->acw, cp->ace));
length = cp->provider->sectorsize;
- offset = cp->provider->mediasize - length;
+ offset = cp->provider->mediasize - length*(1+disk->d_md_offset);
sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
if (md != NULL)
mirror_metadata_encode(md, sector);
@@ -2612,7 +2613,7 @@
#undef DISK_STATE_CHANGED
int
-g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
+g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md, u_int label_offset)
{
struct g_provider *pp;
u_char *buf;
@@ -2626,7 +2627,7 @@
pp = cp->provider;
g_topology_unlock();
/* Metadata are stored on last sector. */
- buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
+ buf = g_read_data(cp, pp->mediasize - pp->sectorsize*(1+label_offset), pp->sectorsize,
&error);
g_topology_lock();
g_access(cp, -1, 0, 0);
@@ -2725,7 +2726,7 @@
int
g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
- struct g_mirror_metadata *md)
+ struct g_mirror_metadata *md, u_int d_md_offset)
{
struct g_mirror_disk *disk;
int error;
@@ -2742,7 +2743,7 @@
pp->name, sc->sc_name);
return (EINVAL);
}
- disk = g_mirror_init_disk(sc, pp, md, &error);
+ disk = g_mirror_init_disk(sc, pp, md, &error, d_md_offset);
if (disk == NULL)
return (error);
error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
@@ -2991,11 +2992,13 @@
struct g_consumer *cp;
struct g_geom *gp;
int error;
+ u_int d_md_offset;
g_topology_assert();
g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
+ d_md_offset = 0;
gp = g_new_geomf(mp, "mirror:taste");
/*
* This orphan function should be never called.
@@ -3003,7 +3006,13 @@
gp->orphan = g_mirror_taste_orphan;
cp = g_new_consumer(gp);
g_attach(cp, pp);
- error = g_mirror_read_metadata(cp, &md);
+ error = g_mirror_read_metadata(cp, &md, 0);
+ if (error != 0) {
+ error = g_mirror_read_metadata(cp, &md, 1);
+ if (error == 0) {
+ d_md_offset = 1;
+ }
+ }
g_detach(cp);
g_destroy_consumer(cp);
g_destroy_geom(gp);
@@ -3057,7 +3066,7 @@
g_topology_unlock();
sx_xlock(&sc->sc_lock);
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
- error = g_mirror_add_disk(sc, pp, &md);
+ error = g_mirror_add_disk(sc, pp, &md, d_md_offset);
if (error != 0) {
G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
pp->name, gp->name, error);
diff -ruN orig/sys/geom/mirror/g_mirror.h new/sys/geom/mirror/g_mirror.h
--- orig/sys/geom/mirror/g_mirror.h 2012-06-16 01:08:42.000000000 +0300
+++ new/sys/geom/mirror/g_mirror.h 2012-06-18 01:22:24.000000000 +0300
@@ -137,6 +137,7 @@
off_t d_last_offset; /* Last read offset */
uint64_t d_flags; /* Additional flags. */
u_int d_genid; /* Disk's generation ID. */
+ u_int d_md_offset; /* metadata offset (in sectors) */
struct g_mirror_disk_sync d_sync;/* Sync information. */
LIST_ENTRY(g_mirror_disk) d_next;
};
@@ -221,8 +222,8 @@
int g_mirror_event_send(void *arg, int state, int flags);
struct g_mirror_metadata;
int g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
- struct g_mirror_metadata *md);
-int g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md);
+ struct g_mirror_metadata *md, u_int d_md_offset);
+int g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md, u_int d_md_offset);
void g_mirror_fill_metadata(struct g_mirror_softc *sc,
struct g_mirror_disk *disk, struct g_mirror_metadata *md);
void g_mirror_update_metadata(struct g_mirror_disk *disk);
diff -ruN orig/sys/geom/mirror/g_mirror_ctl.c new/sys/geom/mirror/g_mirror_ctl.c
--- orig/sys/geom/mirror/g_mirror_ctl.c 2012-06-16 01:08:42.000000000 +0300
+++ new/sys/geom/mirror/g_mirror_ctl.c 2012-06-18 01:23:16.000000000 +0300
@@ -362,7 +362,7 @@
g_mirror_update_metadata(disk);
pp = disk->d_consumer->provider;
g_topology_lock();
- error = g_mirror_read_metadata(disk->d_consumer, &md);
+ error = g_mirror_read_metadata(disk->d_consumer, &md, disk->d_md_offset);
g_topology_unlock();
g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
G_MIRROR_EVENT_WAIT);
@@ -371,7 +371,7 @@
pp->name);
continue;
}
- error = g_mirror_add_disk(sc, pp, &md);
+ error = g_mirror_add_disk(sc, pp, &md, disk->d_md_offset);
if (error != 0) {
gctl_error(req, "Cannot reconnect component %s.",
pp->name);
@@ -390,10 +390,11 @@
struct g_provider *pp;
struct g_consumer *cp;
intmax_t *priority;
+ intmax_t *plabel_offset;
const char *name;
char param[16];
u_char *sector;
- u_int i, n;
+ u_int i, n, label_offset;
int error, *nargs, *hardcode, *inactive;
struct {
struct g_provider *provider;
@@ -424,6 +425,18 @@
gctl_error(req, "No '%s' argument.", "hardcode");
return;
}
+ plabel_offset = gctl_get_paraml(req, "mdoffset", sizeof(*plabel_offset));
+ if (plabel_offset == NULL) {
+ label_offset = 0;
+ return;
+ } else {
+ if (*plabel_offset < 0 || *plabel_offset > 1) {
+ gctl_error(req, "Priority range is 0 to 1, %jd given",
+ *plabel_offset);
+ return;
+ }
+ label_offset = (u_int)(*plabel_offset);
+ }
name = gctl_get_asciiparam(req, "arg0");
if (name == NULL) {
gctl_error(req, "No 'arg%u' argument.", 0);
@@ -512,7 +525,7 @@
sector = g_malloc(pp->sectorsize, M_WAITOK);
mirror_metadata_encode(&md, sector);
error = g_write_data(disks[i].consumer,
- pp->mediasize - pp->sectorsize, sector, pp->sectorsize);
+ pp->mediasize - pp->sectorsize*(1+label_offset), sector, pp->sectorsize);
g_free(sector);
if (error != 0) {
gctl_error(req, "Cannot store metadata on %s.",
@@ -534,6 +547,7 @@
return;
}
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
+ disk->d_md_offset = label_offset;
g_mirror_update_metadata(disk);
}
/*
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201206291039.q5TAdeTL024312>