Date: Thu, 29 Jul 2010 21:23:03 GMT From: Julien Laffaye <jlaffaye@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 181582 for review Message-ID: <201007292123.o6TLN3Td039516@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://p4web.freebsd.org/@@181582?ac=10 Change 181582 by jlaffaye@jlaffaye-chulak on 2010/07/29 21:23:01 - Rework the fetch_archive() function to return a file descriptor of the downloaded archive. Reading on-the-fly doesn't work if we fetch dependencies: the remote server close the connection because we are inactive (in fact we are busy downloading the dependency). Reading on-the-fly could work if we knew the dependencies, in the proper order, of the package before downloading it (thus, download and install the dependencies first). - While I'm here, print the progress (percentage) of the dowload. Affected files ... .. //depot/projects/soc2010/pkg_complete/lib/libpkg/pkg.h#9 edit .. //depot/projects/soc2010/pkg_complete/lib/libpkg/url.c#6 edit .. //depot/projects/soc2010/pkg_complete/usr.sbin/pkg_install/add/perform.c#11 edit Differences ... ==== //depot/projects/soc2010/pkg_complete/lib/libpkg/pkg.h#9 (text+ko) ==== @@ -145,12 +145,6 @@ }; STAILQ_HEAD(reqr_by_head, reqr_by_entry); -struct fetch_data { - FILE *ftp; - int pkgfd; - char buf[8192]; -}; - /* Prototypes */ /* Misc */ int vsystem(const char *, ...); @@ -181,7 +175,7 @@ Boolean isURL(const char *); const char *fileGetURL(const char *, const char *, int); int find_package_url(char * restrict, const char *, const char *); -int fetch_archive(struct archive *, const char *, Boolean); +int fetch_archive(const char *, Boolean); char *fileFindByPath(const char *, const char *); char *fileGetContents(const char *); ssize_t write_file(const char *, const char *); ==== //depot/projects/soc2010/pkg_complete/lib/libpkg/url.c#6 (text+ko) ==== @@ -23,19 +23,13 @@ #include <sys/param.h> #include <sys/wait.h> -#include <archive.h> #include <assert.h> #include <err.h> -#include <errno.h> #include <libgen.h> #include <stdio.h> #include <fetch.h> /* NOTE: stdio must come before fetch. */ #include "pkg.h" -static ssize_t archive_read_cb(struct archive *, void *, const void **); -static int archive_open_cb(struct archive *a, void *); -static int archive_close_cb(struct archive *, void *); - /* * Try and fetch a file by URL, returning the directory name for where * it's unpacked, if successful. @@ -243,123 +237,92 @@ } /* - * Setup the archive `a' callbacks to read data from an URL `url' via fetch(3). - * Returns 0 on success, 1 otherwise. + * Fetch a package archive at `url'. + * Returns a file descriptor of the fetched file, ready to be read. + * On error, returns -1. */ int -fetch_archive(struct archive *a, const char *url, Boolean keep_package) +fetch_archive(const char *url, Boolean keep_package) { - struct fetch_data *data = NULL; - char *tmp; - char pkg[FILENAME_MAX]; - int retcode = 0; + struct url_stat us; + FILE *ftp; + int fd; + int errcode = 0; + char fname[FILENAME_MAX]; + char buf[8192]; + ssize_t r; + off_t done = 0; + int percent; if (!isURL(url)) { warnx("fetch_archive(): '%s' is not an URL!", url); - return (1); + return (-1); } - if ((data = malloc(sizeof(struct fetch_data))) == NULL) - err(EXIT_FAILURE, "malloc()"); - - if (keep_package) { - tmp = getenv("PKGDIR"); - strlcpy(pkg, tmp ? tmp : ".", sizeof(pkg)); - tmp = basename(url); - strlcat(pkg, "/", sizeof(pkg)); - strlcat(pkg, tmp, sizeof(pkg)); - - data->pkgfd = open(pkg, O_WRONLY|O_CREAT|O_TRUNC, 0644); - if (data->pkgfd == -1) { - warn("Error: Unable to open %s", pkg); - retcode = 1; - goto cleanup; - } + if (keep_package == TRUE) { + char *base; + if ((base = getenv("PKGDIR")) == NULL) + base = "."; + snprintf(fname, sizeof(fname), "%s/%s", base, basename(url)); } else - data->pkgfd = 0; + snprintf(fname, sizeof(fname), "/var/tmp/%s", basename(url)); + if (isatty(0) || Verbose) { + printf("Fetching %s...", url); + fflush(stdout); + } + fetchDebug = (Verbose > 0); - if ((data->ftp = fetchGetURL(url, Verbose ? "v" : NULL)) == NULL) { + if ((ftp = fetchXGetURL(url, &us, Verbose ? "v" : NULL)) == NULL) { warnx("Error: Unable to get %s: %s\n", url, fetchLastErrString); - /* If the fetch fails, yank the package. */ - if (keep_package && unlink(pkg) < 0) - warnx("failed to remove partially fetched package: %s", pkg); - retcode = 1; + return (-1); + } + + fd = open(fname, O_RDWR|O_CREAT|O_TRUNC, 0644); + if (fd == -1) { + warn("Error: Unable to open %s", fname); goto cleanup; } - if (isatty(0) || Verbose) { - printf("Fetching %s...", url); + while ((r = fread(buf, 1, sizeof(buf), ftp)) > 0) { + if (write(fd, buf, r) != r) { + warn("write()"); + errcode = -1; + goto cleanup; + } + if (isatty(0)) { + done += r; + percent = ((float)done/us.size) * 100; + printf("\rFetching %s... %d%%", url, percent); fflush(stdout); + } } - - if (archive_read_open(a, data, archive_open_cb, archive_read_cb, - archive_close_cb) != ARCHIVE_OK) { - warnx("Can not open '%s': %s", pkg, archive_error_string(a)); - retcode = 1; + if (ferror(ftp)) { + warn("fread()"); + errcode = -1; goto cleanup; } + printf("\rFetching %s... Done.\n", url); - cleanup: - if (retcode == 1 && data != NULL) - free(data); + /* Reset fd to the beginning of the file */ + lseek(fd, 0, SEEK_SET); - return (retcode); -} + /* + * If we don't want to keep the package, unlink the file now + * so the operating system will delete it for us when we close(2) it. + */ + if (keep_package == FALSE) + unlink(fname); -/* - * Libarchive callback called when more data is needed. - * Read the data from the fetch(3) file descriptor and store it into buf. - * If `pkgfd' is a valid file descriptor, also write the data on disk. - * Returns the read size, 0 on EOF, -1 on error. - */ -static ssize_t -archive_read_cb(struct archive *a, void *client_data, const void **buf) -{ - ssize_t r; - struct fetch_data *data = client_data; - - *buf = data->buf; - if ((r = fread(data->buf, 1, sizeof(data->buf), data->ftp)) < 1) - if (ferror(data->ftp)) { - archive_set_error(a, 0, "error while fetching : %s", - strerror(errno)); - return (-1); + cleanup: + if (ftp != NULL) + fclose(ftp); + if (errcode != 0) { + if (fd > 0) { + close(fd); + unlink(fname); } - - if (data->pkgfd > 0 && r > 0) - if (write(data->pkgfd, buf, r) != r) { - archive_set_error(a, 0, "can not write to package file: %s", - strerror(errno)); - return (-1); - } - - return (r); -} - -/* - * Libarchive callback called by archive_open() - * Since all the job is done in fetch_archive(), always return success. - */ -static int -archive_open_cb(struct archive *a, void *client_data) -{ - return (ARCHIVE_OK); -} - -/* - * Libarchive callback called by archive_close(). - * Release the file descriptors and free the structure. - */ -static int -archive_close_cb(struct archive *a, void *client_data) -{ - struct fetch_data *data = client_data; - - fclose(data->ftp); - if (data->pkgfd > 0) - close(data->pkgfd); - free(data); - - return (ARCHIVE_OK); + fd = -1; + } + return (fd); } ==== //depot/projects/soc2010/pkg_complete/usr.sbin/pkg_install/add/perform.c#11 (text+ko) ==== @@ -80,7 +80,7 @@ * add support for complete packages */ if (isURL(fname)) { - if (fetch_archive(a, fname, KeepPackage) != 0) { + if ((fd = fetch_archive(fname, KeepPackage)) == -1) { warnx("Can not fetch '%s' - aborting", fname); retcode = 1; goto cleanup; @@ -95,13 +95,12 @@ goto cleanup; } } + } - if (archive_read_open_fd(a, fd, 10240) != ARCHIVE_OK) { - warnx("Can not open '%s' archive: %s", fname, - archive_error_string(a)); - retcode = 1; - goto cleanup; - } + if (archive_read_open_fd(a, fd, 10240) != ARCHIVE_OK) { + warnx("archive_read_open_fd(): %s", archive_error_string(a)); + retcode = 1; + goto cleanup; } if (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201007292123.o6TLN3Td039516>