Date: Tue, 11 Jan 2000 22:55:45 +0100 (CET) From: un1i@rz.uni-karlsruhe.de To: FreeBSD-gnats-submit@freebsd.org Subject: misc/16070: [PATCH] Correct and improve HTTP support for sysinstall. Message-ID: <200001112155.WAA60111@i609.hadiko.de>
next in thread | raw e-mail | index | archive | help
>Number: 16070
>Category: misc
>Synopsis: [PATCH] Correct and improve HTTP support for sysinstall.
>Confidential: no
>Severity: serious
>Priority: medium
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Tue Jan 11 14:00:01 PST 2000
>Closed-Date:
>Last-Modified:
>Originator: Philipp Mergenthaler
>Release: FreeBSD 4.0-CURRENT i386
>Organization:
University of Karlsruhe
>Environment:
FreeBSD i609.hadiko.de 4.0-CURRENT FreeBSD 4.0-CURRENT #120: Mon Jan 10 07:34:19 CET 2000 p@i609.hadiko.de:/usr/src/sys/compile/I609 i386
>Description:
Some time ago, the layout in the FTP servers changed. This has to be
fixed in http.c. Also, there were cases (proy/server combinations)
that would result in corruption of those chunks, whose extension
(.ai, .cc, ...) made them look like text files.
>How-To-Repeat:
Install a distribution with at least 9 chunks (so that one has the
extension .ai) via a proxy other than a Squid.
>Fix:
The patch has these changes:
dist.c:
This should catch all cases, were the combination of proxy, server
ans extension leads to chunks treated as text files:
Read each chunk completely into a buffer and compare its length to its
length as given in <distribution>.inf. If they differ, substitute all
CRLFs with LF.
Doing this in distExtract() is easier than in mediaGetHTTP, and it also
might help when you install from DOS or UFS and previously fetched the
files via a proxy.
http.c:
Use the correct FTP path. Adjust the indentation.
media.c:
Slightly reduce verbosity.
sysinstall.8:
Add a paragraph about mediaSetHTTP.
diff -ru sysinstall.orig/dist.c sysinstall/dist.c
--- sysinstall.orig/dist.c Tue Jan 11 19:31:48 2000
+++ sysinstall/dist.c Tue Jan 11 16:15:41 2000
@@ -573,14 +573,15 @@
static Boolean
distExtract(char *parent, Distribution *me)
{
- int i, status, total, intr;
+ int i,j, status, total, intr;
int cpid, zpid, fd2, chunk, numchunks;
- char *path, *dist, buf[BUFSIZ];
+ char *path, *dist, buf[300000];
const char *tmp;
FILE *fp;
WINDOW *w = savescr();
struct timeval start, stop;
struct sigaction old, new;
+ properties dist_attr;
status = TRUE;
if (isDebug())
@@ -643,8 +644,6 @@
}
}
else if (fp > 0) {
- properties dist_attr;
-
if (isDebug())
msgDebug("Parsing attributes file for distribution %s\n", dist);
@@ -661,7 +660,6 @@
numchunks = strtol(tmp, 0, 0);
}
fclose(fp);
- properties_free(dist_attr);
if (!numchunks)
continue;
}
@@ -717,12 +715,19 @@
/* And go for all the chunks */
dialog_clear_norefresh();
for (chunk = 0; chunk < numchunks; chunk++) {
- int n, retval, last_msg;
+ int n, retval, last_msg, chunksize, realsize;
char prompt[80];
last_msg = 0;
getchunk:
+ snprintf(buf, sizeof buf, "cksum.%c%c", (chunk / 26) + 'a', (chunk % 26) + 'a');
+ tmp = property_find(dist_attr, buf);
+ chunksize=0;
+ if (tmp) {
+ tmp=index(tmp, ' ');
+ chunksize = strtol(tmp, 0, 0);
+ }
snprintf(buf, sizeof buf, "%s/%s.%c%c", path, dist, (chunk / 26) + 'a', (chunk % 26) + 'a');
if (isDebug())
msgDebug("trying for piece %d of %d: %s\n", chunk + 1, numchunks, buf);
@@ -744,10 +749,11 @@
snprintf(prompt, sizeof prompt, "Extracting %s into %s directory...", dist, root_bias(me[i].my_dir));
dialog_gauge("Progress", prompt, 8, 15, 6, 50, (int)((float)(chunk + 1) / numchunks * 100));
+ realsize=0;
while (1) {
int seconds;
- n = fread(buf, 1, BUFSIZ, fp);
+ n =fread(buf+realsize, 1, BUFSIZ, fp);
if (check_for_interrupt()) {
msgConfirm("Media read error: User interrupt.");
fclose(fp);
@@ -756,6 +762,7 @@
else if (n <= 0)
break;
total += n;
+ realsize += n;
/* Print statistics about how we're doing */
(void) gettimeofday(&stop, (struct timezone *)0);
@@ -772,19 +779,36 @@
msgInfo("%10d bytes read from %s dist, chunk %2d of %2d @ %.1f KB/sec.",
total, dist, chunk + 1, numchunks, (total / seconds) / 1024.0);
}
- retval = write(fd2, buf, n);
- if (retval != n) {
+ }
+ fclose(fp);
+
+ if (!chunksize || (realsize == chunksize)) {
+ /* No substitution necessary */
+ retval = write(fd2, buf, realsize);
+ if (retval != realsize) {
fclose(fp);
dialog_clear_norefresh();
- msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", retval, n);
+ msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", retval, realsize);
goto punt;
}
+ } else {
+ for(j=0; j<realsize; j++) {
+ /* On finding CRLF, skip the CR; don't exceed end of buffer. */
+ if ((buf[j]!=0x0d) || (j== total-1) || (buf[j+1]!=0x0a)) {
+ retval = write(fd2, buf+j, 1);
+ if (retval != 1) {
+ fclose(fp);
+ dialog_clear_norefresh();
+ msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", j, chunksize);
+ goto punt;
+ }
+ }
+ }
}
- fclose(fp);
}
close(fd2);
status = mediaExtractDistEnd(zpid, cpid);
- goto done;
+ goto done;
punt:
close(fd2);
@@ -814,6 +838,7 @@
else
continue;
}
+ properties_free(dist_attr);
sigaction(SIGINT, &old, NULL); /* Restore signal handler */
restorescr(w);
return status;
diff -ru sysinstall.orig/http.c sysinstall/http.c
--- sysinstall.orig/http.c Tue Jan 4 05:50:59 2000
+++ sysinstall/http.c Tue Jan 11 20:55:10 2000
@@ -11,38 +11,31 @@
mediaInitHTTP(Device *dev)
{
/*
- * Some proxies think that files with the extension ".ai" are postscript
- * files and use "ascii mode" instead of "binary mode" for ftp.
- * The FTP server then translates all LF to CRLF.
- * I don't know how to handle this elegantly...
- * Squid uses ascii mode, ftpget uses binary mode and both tell us:
- * "Content-Type: application/postscript"
- *
- * Probably the safest way would be to get the file, look at its checksum
- * and, if it doesn't match, replace all CRLF by LF and check again.
+ * Some proxies fetch files with certain extensions in "ascii mode" instead
+ * of "binary mode" for FTP. The FTP server then translates all LF to CRLF.
*
* You can force Squid to use binary mode by appending ";type=i" to the URL,
- * which is what I do here.
- *
+ * which is what I do here. For other proxies, the LF->CRLF substitution
+ * is reverted in distExtract().
*/
extern int h_errno;
int rv,s;
- bool el; /* end of header line */
- char *cp, buf[PATH_MAX], req[1000];
+ bool el; /* end of header line */
+ char *cp, buf[PATH_MAX], req[BUFSIZ];
struct sockaddr_in peer;
struct hostent *peer_in;
s=socket(PF_INET, SOCK_STREAM, 6); /* tcp */
if (s == -1) {
- msgConfirm("Network error");
- return FALSE;
+ msgConfirm("Network error");
+ return FALSE;
}
peer_in=gethostbyname(variable_get(VAR_HTTP_HOST));
if (peer_in == NULL) {
- msgConfirm("%s",hstrerror(h_errno));
- return FALSE;
+ msgConfirm("%s",hstrerror(h_errno));
+ return FALSE;
}
peer.sin_len=peer_in->h_length;
@@ -52,9 +45,9 @@
rv=connect(s,(struct sockaddr *)&peer,sizeof(peer));
if (rv == -1) {
- msgConfirm("Couldn't connect to proxy %s:%s",
- variable_get(VAR_HTTP_HOST),variable_get(VAR_FTP_PORT));
- return FALSE;
+ msgConfirm("Couldn't connect to proxy %s:%s",
+ variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
+ return FALSE;
}
sprintf(req,"GET / HTTP/1.0\r\n\r\n");
@@ -69,28 +62,28 @@
rv=read(s,cp,1);
variable_set2(VAR_HTTP_FTP_MODE,"",0);
while (rv>0) {
- if ((*cp == '\012') && el) {
- /* reached end of a header line */
- if (!strncmp(buf,"Server: ",8)) {
- if (!strncmp(buf,"Server: Squid",13)) {
- variable_set2(VAR_HTTP_FTP_MODE,";type=i",1);
- } else {
- variable_set2(VAR_HTTP_FTP_MODE,"",1);
- }
- }
- /* ignore other headers */
- /* check for "\015\012" at beginning of line, i.e. end of headers */
- if ((cp-buf) == 1)
- break;
- cp=buf;
- rv=read(s,cp,1);
- } else {
- el=FALSE;
- if (*cp == '\015')
- el=TRUE;
- cp++;
- rv=read(s,cp,1);
- }
+ if ((*cp == '\012') && el) {
+ /* reached end of a header line */
+ if (!strncmp(buf,"Server: ",8)) {
+ if (!strncmp(buf,"Server: Squid",13)) {
+ variable_set2(VAR_HTTP_FTP_MODE,";type=i",0);
+ } else {
+ variable_set2(VAR_HTTP_FTP_MODE,"",0);
+ }
+ }
+ /* ignore other headers */
+ /* check for "\015\012" at beginning of line, i.e. end of headers */
+ if ((cp-buf) == 1)
+ break;
+ cp=buf;
+ rv=read(s,cp,1);
+ } else {
+ el=FALSE;
+ if (*cp == '\015')
+ el=TRUE;
+ cp++;
+ rv=read(s,cp,1);
+ }
}
close(s);
return TRUE;
@@ -103,14 +96,14 @@
FILE *fp;
int rv,s;
bool el; /* end of header line */
- char *cp, buf[PATH_MAX], req[1000];
+ char *cp, buf[PATH_MAX], req[BUFSIZ];
struct sockaddr_in peer;
struct hostent *peer_in;
s=socket(PF_INET, SOCK_STREAM, 6); /* tcp */
if (s == -1) {
- msgConfirm("Network error");
- return NULL;
+ msgConfirm("Network error");
+ return NULL;
}
peer_in=gethostbyname(variable_get(VAR_HTTP_HOST));
@@ -121,16 +114,18 @@
rv=connect(s,(struct sockaddr *)&peer,sizeof(peer));
if (rv == -1) {
- msgConfirm("Couldn't connect to proxy %s:%s",
- variable_get(VAR_HTTP_HOST),variable_get(VAR_FTP_PORT));
- return NULL;
- }
-
- sprintf(req,"GET ftp://%s:%s%s%s/%s%s HTTP/1.0\r\n\r\n",
- variable_get(VAR_FTP_HOST), variable_get(VAR_FTP_PORT),
- "/pub/FreeBSD/", variable_get(VAR_RELNAME),
- file,variable_get(VAR_HTTP_FTP_MODE));
- msgDebug("sending http request: %s",req);
+ msgConfirm("Couldn't connect to proxy %s:%s",
+ variable_get(VAR_HTTP_HOST),variable_get(VAR_FTP_PORT));
+ return NULL;
+ }
+
+ sprintf(req,"GET %s/%s/%s%s HTTP/1.0\r\n\r\n",
+ variable_get(VAR_FTP_PATH), variable_get(VAR_RELNAME),
+ file, variable_get(VAR_HTTP_FTP_MODE));
+
+ if (isDebug()) {
+ msgDebug("sending http request: %s",req);
+ }
write(s,req,strlen(req));
/*
@@ -142,41 +137,43 @@
el=FALSE;
rv=read(s,cp,1);
while (rv>0) {
- if ((*cp == '\012') && el) {
- /* reached end of a header line */
- if (!strncmp(buf,"HTTP",4)) {
- rv=strtol((char *)(buf+9),0,0);
- *(cp-1)='\0'; /* chop the CRLF off */
- if (rv >= 500) {
- msgConfirm("Server error %s, you could try an other server",buf);
- return NULL;
- } else if (rv == 404) {
- msgConfirm("%s was not found, maybe directory or release-version are wrong?",req);
- return NULL;
- } else if (rv >= 400) {
- msgConfirm("Client error %s, you could try an other server",buf);
- return NULL;
- } else if (rv >= 300) {
- msgConfirm("Error %s,",buf);
- return NULL;
- } else if (rv != 200) {
- msgConfirm("Error %s when trying to fetch %s",buf,req);
- return NULL;
- }
- }
- /* ignore other headers */
- /* check for "\015\012" at beginning of line, i.e. end of headers */
- if ((cp-buf) == 1)
- break;
- cp=buf;
- rv=read(s,cp,1);
- } else {
- el=FALSE;
- if (*cp == '\015')
- el=TRUE;
- cp++;
- rv=read(s,cp,1);
- }
+ if ((*cp == '\012') && el) {
+ /* reached end of a header line */
+ if (!strncmp(buf,"HTTP",4)) {
+ rv=strtol((char *)(buf+9),0,0);
+ *(cp-1)='\0'; /* chop the CRLF off */
+ if (probe && (rv != 200)) {
+ return NULL;
+ } else if (rv >= 500) {
+ msgConfirm("Server error %s when sending %s, you could try an other server",buf, req);
+ return NULL;
+ } else if (rv == 404) {
+ msgConfirm("%s was not found, maybe directory or release-version are wrong?",req);
+ return NULL;
+ } else if (rv >= 400) {
+ msgConfirm("Client error %s, you could try an other server",buf);
+ return NULL;
+ } else if (rv >= 300) {
+ msgConfirm("Error %s,",buf);
+ return NULL;
+ } else if (rv != 200) {
+ msgConfirm("Error %s when sending %s, you could try an other server",buf, req);
+ return NULL;
+ }
+ }
+ /* ignore other headers */
+ /* check for "\015\012" at beginning of line, i.e. end of headers */
+ if ((cp-buf) == 1)
+ break;
+ cp=buf;
+ rv=read(s,cp,1);
+ } else {
+ el=FALSE;
+ if (*cp == '\015')
+ el=TRUE;
+ cp++;
+ rv=read(s,cp,1);
+ }
}
fp=fdopen(s,"r");
return fp;
diff -ru sysinstall.orig/media.c sysinstall/media.c
--- sysinstall.orig/media.c Tue Jan 11 21:00:42 2000
+++ sysinstall/media.c Tue Jan 11 22:03:25 2000
@@ -445,7 +445,6 @@
if (DITEM_STATUS(result) != DITEM_SUCCESS)
return result;
- variable_set2(VAR_HTTP_PATH, "", 0);
cp = variable_get_value(VAR_HTTP_PATH,
"Please enter the address of the HTTP proxy in this format:\n"
" hostname:port (the ':port' is optional, default is 3128)",0);
@@ -461,11 +460,11 @@
variable_set2(VAR_HTTP_HOST, hostname, 0);
variable_set2(VAR_HTTP_PORT, itoa(HttpPort), 0);
- msgDebug("VAR_HTTP_HOST, _PORT: %s:%s",variable_get(VAR_HTTP_HOST),
- variable_get(VAR_HTTP_PORT));
-
- msgDebug("VAR_FTP_HOST, _PORT: %s:%s", variable_get(VAR_FTP_HOST),
- variable_get(VAR_FTP_PORT));
+ if (isDebug()) {
+ msgDebug("VAR_FTP_PATH : %s",variable_get(VAR_FTP_PATH));
+ msgDebug("VAR_HTTP_HOST, _PORT: %s:%s",variable_get(VAR_HTTP_HOST),
+ variable_get(VAR_HTTP_PORT));
+ }
/* mediaDevice has been set by mediaSetFTP(), overwrite partly: */
mediaDevice->type = DEVICE_TYPE_HTTP;
diff -ru sysinstall.orig/sysinstall.8 sysinstall/sysinstall.8
--- sysinstall.orig/sysinstall.8 Tue Jan 11 07:04:50 2000
+++ sysinstall/sysinstall.8 Tue Jan 11 21:55:23 2000
@@ -675,6 +675,9 @@
Select a tape device as the installation media.
.Pp
\fBVariables:\fR None
+
+.\" XXX
+
.It mediaSetFTP
Select an FTP site as the installation media.
.Pp
@@ -719,6 +722,18 @@
.Pp
\fBVariables:\fR Same as for
.Ar mediaSetFTP .
+.It mediaSetHTTP
+Alias for
+.Ar mediaSetFTP
+using an HTTP proxy.
+.Pp
+\fBVariables:\fR See
+.Ar mediaSetFTP ,
+plus
+.Bl -tag -width indent
+.It _httpPath
+The proxy to use (host:port) (non-optional).
+.El
.It mediaSetUFS
Select an existing UFS partition (mounted with the label editor) as
the installation media.
>Release-Note:
>Audit-Trail:
>Unformatted:
To Unsubscribe: send mail to majordomo@FreeBSD.org
with "unsubscribe freebsd-bugs" in the body of the message
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200001112155.WAA60111>