From owner-p4-projects@FreeBSD.ORG Fri Jun 4 00:50:47 2010 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 45F3C1065670; Fri, 4 Jun 2010 00:50:47 +0000 (UTC) Delivered-To: perforce@FreeBSD.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 0A728106566B for ; Fri, 4 Jun 2010 00:50:47 +0000 (UTC) (envelope-from ivoras@FreeBSD.org) Received: from repoman.freebsd.org (unknown [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id EBE058FC0A for ; Fri, 4 Jun 2010 00:50:46 +0000 (UTC) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.3/8.14.3) with ESMTP id o540okpT078846 for ; Fri, 4 Jun 2010 00:50:46 GMT (envelope-from ivoras@FreeBSD.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.3/8.14.3/Submit) id o540okCk078844 for perforce@freebsd.org; Fri, 4 Jun 2010 00:50:46 GMT (envelope-from ivoras@FreeBSD.org) Date: Fri, 4 Jun 2010 00:50:46 GMT Message-Id: <201006040050.o540okCk078844@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to ivoras@FreeBSD.org using -f From: Ivan Voras To: Perforce Change Reviews Precedence: bulk Cc: Subject: PERFORCE change 179161 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 04 Jun 2010 00:50:47 -0000 http://p4web.freebsd.org/@@179161?ac=10 Change 179161 by ivoras@betelgeuse on 2010/06/04 00:50:15 Add support for bsdiffed patches ("-b") Affected files ... .. //depot/projects/soc2010/pkg_patch/src/patch/Makefile#10 edit .. //depot/projects/soc2010/pkg_patch/src/patch/hashjob.c#9 edit .. //depot/projects/soc2010/pkg_patch/src/patch/hashjob.h#9 edit .. //depot/projects/soc2010/pkg_patch/src/patch/main.c#10 edit .. //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.c#8 edit .. //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.h#8 edit .. //depot/projects/soc2010/pkg_patch/src/patch/pkg_patch.h#8 edit .. //depot/projects/soc2010/pkg_patch/src/patch/support.c#7 edit Differences ... ==== //depot/projects/soc2010/pkg_patch/src/patch/Makefile#10 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/hashjob.c#9 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/hashjob.h#9 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/main.c#10 (text+ko) ==== @@ -38,7 +38,7 @@ char **argv; enum PP_OP patch_op = PP_NONE; char *my_tmp = NULL; -int verbose = 0; +int flag_bsdiff = 0; static void usage_short(void); @@ -49,7 +49,7 @@ static void usage_short() { - printf("usage: %s -m [flags] package_file_1 package_file_2\n", argv[0]); + printf("usage: %s -c [-b] package_file_1 package_file_2\n", argv[0]); } @@ -57,9 +57,12 @@ proc_args() { int ch; - while ((ch = getopt(argc, argv, "hmv")) != -1) { + while ((ch = getopt(argc, argv, "bchv")) != -1) { switch (ch) { - case 'm': + case 'b': + flag_bsdiff = 1; + break; + case 'c': patch_op = PP_MKPATCH; break; case 'h': @@ -67,7 +70,7 @@ exit(0); break; case 'v': - verbose++; + Verbose++; break; default: usage_short(); @@ -93,7 +96,7 @@ asprintf(&my_tmp, "%spkg_patch.%d.%d", _PATH_TMP, getpid(), time(NULL)); if (mkdir(my_tmp, 0700) != 0) errx(1, "Cannot create working directory: %s", my_tmp); - if (verbose) + if (Verbose) printf("Using temporary directory: %s\n", my_tmp); } ==== //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.c#8 (text+ko) ==== @@ -40,11 +40,13 @@ char fold[PATH_MAX], fnew[PATH_MAX], fpatch[PATH_MAX]; char dold[PATH_MAX], dnew[PATH_MAX], dpatch[PATH_MAX]; char tmp[PATH_MAX], tmp2[PATH_MAX]; + const char *method; struct pkgxjob xold, xnew; struct filelist_head flold, flnew; struct filelist_head fldiff_old_new, fldiff_new_old, flintersect; struct filelist_head flchanged; struct filelist *fl; + unsigned n_changed_files; FILE *fp; time_t tm; @@ -90,7 +92,7 @@ filelist_gather(dold, &flold); SLIST_INIT(&flnew); filelist_gather(dnew, &flnew); - if (verbose) + if (Verbose) printf("Processing %d files in old package and %d in new.\n", filelist_count(&flold), filelist_count(&flnew)); @@ -100,17 +102,17 @@ filelist_diff(&flnew, &flold, &fldiff_new_old); SLIST_INIT(&flintersect); filelist_intersect(&flnew, &flold, &flintersect); - if (verbose) + if (Verbose) printf("Found %d files to add and %d files to delete.\n", filelist_count(&fldiff_new_old), filelist_count(&fldiff_old_new)); - if (verbose > 2) { + if (Verbose > 2) { SLIST_FOREACH(fl, &fldiff_new_old, linkage) printf("++ %s\n", fl->filename); SLIST_FOREACH(fl, &fldiff_old_new, linkage) printf("-- %s\n", fl->filename); - if (verbose > 3) + if (Verbose > 3) SLIST_FOREACH(fl, &flintersect, linkage) printf("?? %s\n", fl->filename); } @@ -152,15 +154,19 @@ hjold_md5.hash_len) != 0 || memcmp(hjold_sha256.hash, hjnew_sha256.hash, hjold_sha256.hash_len) != 0) { /* Assume changed files */ - if (verbose > 3) + if (Verbose > 3) printf("~~ %s\n", fl->filename); fl2 = malloc(sizeof(*fl2)); memcpy(fl2, fl, sizeof(*fl2)); SLIST_INSERT_HEAD(&flchanged, fl2, linkage); } } - if (verbose) + if (Verbose) printf("Found %d changed files.\n", filelist_count(&flchanged)); + if (flag_bsdiff) + method = "bsdiff"; + else + method = "cp"; /* * XXX: Possibly reimplement with libarchive. @@ -185,9 +191,13 @@ SLIST_FOREACH(fl, &fldiff_old_new, linkage) if (S_ISDIR(fl->st.st_mode)) fprintf(fp, "@rmdir %s\n", fl->filename); + n_changed_files = 0; SLIST_FOREACH(fl, &flchanged, linkage) - if (fl->filename[0] != '+') - fprintf(fp, "@patch [method=cp] %s\n", fl->filename); + if (fl->filename[0] != '+') { + fprintf(fp, "@patch [method=%s] %s\n", method, + fl->filename); + n_changed_files++; + } if (fclose(fp) != 0) err(1, "Cannot close %s", PKGPATCH_FNAME); @@ -214,14 +224,66 @@ err(1, "Cannot copy file: %s to file: %s", tmp, tmp2); } - /* Copy changed files */ - SLIST_FOREACH(fl, &flchanged, linkage) { - snprintf(tmp, PATH_MAX, "%s/%s", dnew, fl->filename); - snprintf(tmp2, PATH_MAX, "%s/%s", dpatch, fl->filename); - if (copy_file_absolute(tmp, tmp2) != 0) - err(1, "Cannot copy file: %s to file: %s", tmp, tmp2); + /* Handle changed files */ + if (!flag_bsdiff) { + SLIST_FOREACH(fl, &flchanged, linkage) { + if (fl->filename[0] == '+') + continue; + snprintf(tmp, PATH_MAX, "%s/%s", dnew, fl->filename); + snprintf(tmp2, PATH_MAX, "%s/%s", dpatch, fl->filename); + if (copy_file_absolute(tmp, tmp2) != 0) + err(1, "Cannot copy file: %s to file: %s", tmp, + tmp2); + } + } else { + /* + * bsdiff is extremely CPU-intensive, but multiple invocations + * can be started so it becomes embarrasingly parallel on SMP. + * I've observed linear or better processing time improvments + * with this simple trick. + */ + FILE **fplist = calloc(n_changed_files, sizeof(*fplist)); + int n = 0; + + if (fplist == NULL) + err(1, "calloc() failed"); + /* Start jobs */ + SLIST_FOREACH(fl, &flchanged, linkage) { + if (fl->filename[0] == '+') + continue; + if (Verbose > 1) + printf("bsdiff for %s\n", fl->filename); + snprintf(tmp, PATH_MAX, "%s %s/%s %s/%s %s/%s.bsdiff", + _PATH_BSDIFF, + dold, fl->filename, + dnew, fl->filename, + dpatch, fl->filename); + fplist[n] = popen(tmp, "r+"); + if (fplist[n] == NULL) + err(1, "Cannot popen bsdiff for %s", + fl->filename); + n++; + } + /* Collect jobs */ + n = 0; + SLIST_FOREACH(fl, &flchanged, linkage) { + if (fl->filename[0] == '+') + continue; + if (pclose(fplist[n]) < 0) + err(1, "pclose() failed for bsdiff of %s", + fl->filename); + n++; + snprintf(tmp, PATH_MAX, "%s/%s", dold, fl->filename); + snprintf(tmp2, PATH_MAX, "%s/%s.bsdiff", dpatch, + fl->filename); + if (copy_file_attrs(tmp, NULL, tmp2) != 0) + err(1, "copy_file_attrs(%s,%s) failed", + tmp, tmp2); + } + free(fplist); } + /* Finally, create the patch archive and call it a day. */ chdir(dpatch); snprintf(tmp, PATH_MAX, "%s -c -j -f %s *", _PATH_TAR, fpatch); fp = popen(tmp, "r+"); @@ -230,5 +292,7 @@ rm_rf(dold); rm_rf(dnew); if (pclose(fp) != 0) - err(1, "pclose() failed"); + err(1, "pclose() failed on final tar"); + if (Verbose) + printf("Created %s.\n", fpatch); } ==== //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.h#8 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/pkg_patch.h#8 (text+ko) ==== @@ -23,6 +23,9 @@ #ifndef _PATH_TAR #define _PATH_TAR "/usr/bin/tar" #endif +#ifndef _PATH_BSDIFF +#define _PATH_BSDIFF "/usr/bin/bsdiff" +#endif #define PKGPATCH_FNAME "+PKGPATCH" #define PKGPATCH_VERSION "1.0" @@ -47,7 +50,7 @@ extern char **argv; extern enum PP_OP patch_op; extern char *my_tmp; -extern int verbose; +extern int flag_bsdiff; #endif @@ -63,6 +66,7 @@ void parse_package_name(char *pkgfile, char *basename, char *version, char *suffix); int copy_file_absolute(char *from, char *to); +int copy_file_attrs(char *from, struct stat *st_from, char *to); int replicate_dirtree(char *from, char *to); #endif ==== //depot/projects/soc2010/pkg_patch/src/patch/support.c#7 (text+ko) ==== @@ -37,9 +37,6 @@ #include "pkg_patch.h" -static int copy_file_attrs(char *from, struct stat *st_from, char *to); - - int rm_rf(char *dir) { @@ -59,7 +56,7 @@ /* libarchive not threadsafe for extract; call external tar */ job->filename = filename; sprintf(cmd, "%s -x -C %s -f %s", _PATH_TAR, dir, filename); - if (verbose > 1) + if (Verbose > 1) printf("pkgxjob: %s\n", cmd); job->fp = popen(cmd, "r+"); if (job->fp == NULL) @@ -96,6 +93,8 @@ if (fe->fts_pathlen == dir_len) continue; fl = malloc(sizeof(*fl)); + if (fl == NULL) + return (-1); strncpy(fl->filename, fe->fts_path + dir_len + 1, PATH_MAX); memcpy(&fl->st, fe->fts_statp, sizeof(struct stat)); @@ -206,7 +205,7 @@ * Copy generic file attributes. * TODO: See if there is any need to take care of ACLs (tar apparently doesn't). */ -static int +int copy_file_attrs(char *from, struct stat *st_from, char *to) { struct stat *st, st2;