From owner-svn-src-head@FreeBSD.ORG Mon Dec 15 08:27:45 2008 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 871841065672; Mon, 15 Dec 2008 08:27:45 +0000 (UTC) (envelope-from murray@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 74F248FC12; Mon, 15 Dec 2008 08:27:45 +0000 (UTC) (envelope-from murray@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id mBF8RjGV089098; Mon, 15 Dec 2008 08:27:45 GMT (envelope-from murray@svn.freebsd.org) Received: (from murray@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id mBF8RjFB089093; Mon, 15 Dec 2008 08:27:45 GMT (envelope-from murray@svn.freebsd.org) Message-Id: <200812150827.mBF8RjFB089093@svn.freebsd.org> From: Murray Stokely Date: Mon, 15 Dec 2008 08:27:45 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r186124 - in head: lib/libfetch usr.bin/fetch X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 15 Dec 2008 08:27:45 -0000 Author: murray Date: Mon Dec 15 08:27:44 2008 New Revision: 186124 URL: http://svn.freebsd.org/changeset/base/186124 Log: Add support for HTTP 1.1 If-Modified-Since behavior. fetch(1) accepts a new argument -i that if specified will cause the file to be downloaded only if it is more recent than the mtime of . libfetch(3) accepts the mtime in the url structure and a flag to indicate when this behavior is desired. PR: bin/87841 Submitted by: Jukka A. Ukkonen (partially) Reviewed by: des, ru MFC after: 3 weeks Modified: head/lib/libfetch/fetch.3 head/lib/libfetch/fetch.h head/lib/libfetch/http.c head/usr.bin/fetch/fetch.1 head/usr.bin/fetch/fetch.c Modified: head/lib/libfetch/fetch.3 ============================================================================== --- head/lib/libfetch/fetch.3 Mon Dec 15 08:11:45 2008 (r186123) +++ head/lib/libfetch/fetch.3 Mon Dec 15 08:27:44 2008 (r186124) @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd December 18, 2007 +.Dd December 14, 2008 .Dt FETCH 3 .Os .Sh NAME @@ -165,9 +165,16 @@ struct url { char *doc; off_t offset; size_t length; + time_t ims_time; }; .Ed .Pp +The +.Va ims_time +field stores the time value for +.Li If-Modified-Since +HTTP requests. +.Pp The pointer returned by .Fn fetchMakeURL or @@ -353,6 +360,22 @@ and .Fn fetchPutHTTP will use a direct connection even if a proxy server is defined. .Pp +If the +.Ql i +(if-modified-since) flag is specified, and +the +.Va ims_time +field is set in +.Vt "struct url" , +then +.Fn fetchXGetHTTP +and +.Fn fetchGetHTTP +will send a conditional +.Li If-Modified-Since +HTTP header to only fetch the content if it is newer than +.Va ims_time . +.Pp Since there seems to be no good way of implementing the HTTP PUT method in a manner consistent with the rest of the .Nm fetch Modified: head/lib/libfetch/fetch.h ============================================================================== --- head/lib/libfetch/fetch.h Mon Dec 15 08:11:45 2008 (r186123) +++ head/lib/libfetch/fetch.h Mon Dec 15 08:27:44 2008 (r186124) @@ -46,6 +46,7 @@ struct url { char *doc; off_t offset; size_t length; + time_t ims_time; }; struct url_stat { Modified: head/lib/libfetch/http.c ============================================================================== --- head/lib/libfetch/http.c Mon Dec 15 08:11:45 2008 (r186123) +++ head/lib/libfetch/http.c Mon Dec 15 08:27:44 2008 (r186124) @@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include @@ -92,6 +93,7 @@ __FBSDID("$FreeBSD$"); #define HTTP_MOVED_PERM 301 #define HTTP_MOVED_TEMP 302 #define HTTP_SEE_OTHER 303 +#define HTTP_NOT_MODIFIED 304 #define HTTP_TEMP_REDIRECT 307 #define HTTP_NEED_AUTH 401 #define HTTP_NEED_PROXY_AUTH 407 @@ -797,20 +799,23 @@ FILE * http_request(struct url *URL, const char *op, struct url_stat *us, struct url *purl, const char *flags) { + char timebuf[80]; + char hbuf[MAXHOSTNAMELEN + 7], *host; conn_t *conn; struct url *url, *new; - int chunked, direct, need_auth, noredirect, verbose; + int chunked, direct, ims, need_auth, noredirect, verbose; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; const char *p; FILE *f; hdr_t h; - char hbuf[MAXHOSTNAMELEN + 7], *host; + struct tm *timestruct; direct = CHECK_FLAG('d'); noredirect = CHECK_FLAG('A'); verbose = CHECK_FLAG('v'); + ims = CHECK_FLAG('i'); if (direct && purl) { fetchFreeURL(purl); @@ -879,6 +884,14 @@ http_request(struct url *URL, const char op, url->doc); } + if (ims && url->ims_time) { + timestruct = gmtime((time_t *)&url->ims_time); + (void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT", + timestruct); + if (verbose) + fetch_info("If-Modified-Since: %s", timebuf); + http_cmd(conn, "If-Modified-Since: %s", timebuf); + } /* virtual host */ http_cmd(conn, "Host: %s", host); @@ -940,6 +953,7 @@ http_request(struct url *URL, const char switch (http_get_reply(conn)) { case HTTP_OK: case HTTP_PARTIAL: + case HTTP_NOT_MODIFIED: /* fine */ break; case HTTP_MOVED_PERM: @@ -1074,7 +1088,10 @@ http_request(struct url *URL, const char } /* we have a hit or an error */ - if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err)) + if (conn->err == HTTP_OK + || conn->err == HTTP_NOT_MODIFIED + || conn->err == HTTP_PARTIAL + || HTTP_ERROR(conn->err)) break; /* all other cases: we got a redirect */ @@ -1102,6 +1119,11 @@ http_request(struct url *URL, const char (long long)offset, (long long)length, (long long)size, (long long)clength)); + if (conn->err == HTTP_NOT_MODIFIED) { + http_seterr(HTTP_NOT_MODIFIED); + return (NULL); + } + /* check for inconsistencies */ if (clength != -1 && length != -1 && clength != length) { http_seterr(HTTP_PROTOCOL_ERROR); Modified: head/usr.bin/fetch/fetch.1 ============================================================================== --- head/usr.bin/fetch/fetch.1 Mon Dec 15 08:11:45 2008 (r186123) +++ head/usr.bin/fetch/fetch.1 Mon Dec 15 08:27:44 2008 (r186124) @@ -29,7 +29,7 @@ .\" .\" $FreeBSD$ .\" -.Dd March 11, 2003 +.Dd December 14, 2008 .Dt FETCH 1 .Os .Sh NAME @@ -39,6 +39,7 @@ .Nm .Op Fl 146AadFlMmnPpqRrsUv .Op Fl B Ar bytes +.Op Fl i Ar file .Op Fl N Ar file .Op Fl o Ar file .Op Fl S Ar bytes @@ -48,6 +49,7 @@ .Nm .Op Fl 146AadFlMmnPpqRrsUv .Op Fl B Ar bytes +.Op Fl i Ar file .Op Fl N Ar file .Op Fl o Ar file .Op Fl S Ar bytes @@ -116,6 +118,12 @@ The file to retrieve is located on the h .Ar host . This option is deprecated and is provided for backward compatibility only. +.It Fl i Ar file +If-Modified-Since mode: the remote file will only be retrieved if it +is newer than +.Ar file +on the local host. +(HTTP only) .It Fl l If the target is a file-scheme URL, make a symbolic link to the target rather than trying to copy it. @@ -249,6 +257,12 @@ If multiple URLs are listed on the comma .Nm will attempt to retrieve each one of them in turn, and will return zero only if they were all successfully retrieved. +.Pp +If the +.Fl i +argument is used and the remote file is not newer than the +specified file then the command will still return success, +although no file is transferred. .Sh SEE ALSO .Xr fetch 3 .Sh HISTORY Modified: head/usr.bin/fetch/fetch.c ============================================================================== --- head/usr.bin/fetch/fetch.c Mon Dec 15 08:11:45 2008 (r186123) +++ head/usr.bin/fetch/fetch.c Mon Dec 15 08:27:44 2008 (r186124) @@ -60,6 +60,8 @@ int d_flag; /* -d: direct connection int F_flag; /* -F: restart without checking mtime */ char *f_filename; /* -f: file to fetch */ char *h_hostname; /* -h: host to fetch from */ +int i_flag; /* -i: specify input file for mtime comparison */ +char *i_filename; /* name of input file */ int l_flag; /* -l: link rather than copy file: URLs */ int m_flag; /* -[Mm]: mirror mode */ char *N_filename; /* -N: netrc file name */ @@ -382,6 +384,14 @@ fetch(char *URL, const char *path) if (A_flag) strcat(flags, "A"); timeout = T_secs ? T_secs : http_timeout; + if (i_flag) { + if (stat(i_filename, &sb)) { + warn("%s: stat()", i_filename); + goto failure; + } + url->ims_time = sb.st_mtime; + strcat(flags, "i"); + } } /* set the protocol timeout. */ @@ -449,7 +459,14 @@ fetch(char *URL, const char *path) goto signal; if (f == NULL) { warnx("%s: %s", URL, fetchLastErrString); - goto failure; + if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0 + && fetchLastErrCode == FETCH_OK + && strcmp(fetchLastErrString, "Not Modified") == 0) { + /* HTTP Not Modified Response, return OK. */ + r = 0; + goto done; + } else + goto failure; } if (sigint) goto signal; @@ -713,9 +730,9 @@ usage(void) { fprintf(stderr, "%s\n%s\n%s\n%s\n", "usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", -" [-T seconds] [-w seconds] URL ...", +" [-T seconds] [-w seconds] [-i file] URL ...", " fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", -" [-T seconds] [-w seconds] -h host -f file [-c dir]"); +" [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]"); } @@ -732,7 +749,7 @@ main(int argc, char *argv[]) int c, e, r; while ((c = getopt(argc, argv, - "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) + "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) switch (c) { case '1': once_flag = 1; @@ -777,6 +794,10 @@ main(int argc, char *argv[]) case 'h': h_hostname = optarg; break; + case 'i': + i_flag = 1; + i_filename = optarg; + break; case 'l': l_flag = 1; break;