Date: Fri, 28 Mar 2003 22:47:43 -0800 From: Tim Kientzle <kientzle@acm.org> To: "Bruce A. Mah" <bmah@FreeBSD.ORG> Cc: freebsd-hackers@FreeBSD.ORG Subject: Re: Making pkg_XXX tools smarter about file types... Message-ID: <3E85418F.8010201@acm.org> References: <3E42C148.4050807@acm.org> <20030329012828.GA32891@intruder.bmah.org>
next in thread | previous in thread | raw e-mail | index | archive | help
This is a multi-part message in MIME format.
--------------070001080502070007020104
Content-Type: text/plain; charset=us-ascii; format=flowed
Content-Transfer-Encoding: 7bit
Bruce A. Mah wrote:
> If memory serves me right, Tim Kientzle wrote:
>
>>The attached patch modifies the pkg_install
>>tools to inspect the file contents--rather than the
>>filename extension--to determine the
>>compression method in use.
>
> The concept is good, and it's something we've needed for awhile. I
> suspect you followed the various adventures of pkg_add and sysinstall
> when we tried supporting both bzip2 and gzip packages ...
Yes, those adventures were a large part of
what motivated me to implement this auto-detect logic.
> ... If the
> package comes from a file, then pkg_add wants to make two passes over
> the package, first to get the +CONTENTS file and second to actually
> unpack everything. When the first tar process finishes reading the
> +CONTENTS file, it closes its pipe (due to the --fast-read argument).
> However, pkg_add still seems to be writing to the pipe...this seems to
> be bad.
>
> It works if I remove the --fast-read flag from the tar, but that's not
> the right answer.
No. That's clearly not the right answer. Seems I forgot
to check for an error return from fwrite(). Easy enough
to fix; near the end of unpack() in lib/file.c, change
the read/write loop to the following:
while(buff_size > 0) {
if(buff_size < fwrite(buff,1,buff_size,out_pipe))
break;
buff_size = fread(buff,1,buff_allocation,pkg_file);
}
This aborts the passthrough if the pipe is closed.
Modified diff attached.
Tim
P.S. It's galled me for a while that pkg_add has
to fork 'tar' to extract the archive. I've started
piecing together a library that reads/writes tarfiles.
With this, it should be possible to make pkg_add
considerably more efficient. In particular, rather
than extracting to a temp directory, then parsing important
information, then moving the files, it should be
possible using this library to read the initial
entries ("+CONTENTS", in particular) directly into
memory, process the information there, then extract the
remainder of the package files directly into their
final locations. So far, I have a library API
outlined, and functional read support implemented.
Next step is to hack up a minimal tar implementation
that uses it to make sure everything's working
correctly.
So far, the library automatically detects compression
formats (using techniques like those in my
pkg_install patch) and has some rough support
for detecting the archive format as well.
(One goal of mine: support for 'pax extended
archives', which I understand can handle ACLs.)
Of course, such a library could also form the
basis for a BSD-licensed tar to replace GNU tar.
I understand a few people have wanted such a thing.
--------------070001080502070007020104
Content-Type: text/plain;
name="kientzle_pkg_install-2.diff"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="kientzle_pkg_install-2.diff"
Index: lib/exec.c
===================================================================
RCS file: /usr/src/cvs/src/usr.sbin/pkg_install/lib/exec.c,v
retrieving revision 1.10
diff -c -r1.10 exec.c
*** lib/exec.c 1 Apr 2002 09:39:07 -0000 1.10
--- lib/exec.c 29 Mar 2003 06:00:36 -0000
***************
*** 25,59 ****
#include <err.h>
/*
! * Unusual system() substitute. Accepts format string and args,
! * builds and executes command. Returns exit code.
*/
!
! int
! vsystem(const char *fmt, ...)
{
! va_list args;
char *cmd;
- int ret, maxargs;
maxargs = sysconf(_SC_ARG_MAX);
maxargs -= 32; /* some slop for the sh -c */
cmd = malloc(maxargs);
if (!cmd) {
! warnx("vsystem can't alloc arg space");
! return 1;
}
-
- va_start(args, fmt);
if (vsnprintf(cmd, maxargs, fmt, args) > maxargs) {
! warnx("vsystem args are too long");
! return 1;
}
#ifdef DEBUG
printf("Executing %s\n", cmd);
#endif
ret = system(cmd);
- va_end(args);
free(cmd);
return ret;
}
--- 25,83 ----
#include <err.h>
/*
! * Format a command, allocating a buffer along the way.
*/
! static char *
! va_system_cmd(const char *fmt, va_list args)
{
! int maxargs;
char *cmd;
maxargs = sysconf(_SC_ARG_MAX);
maxargs -= 32; /* some slop for the sh -c */
cmd = malloc(maxargs);
if (!cmd) {
! warnx("can't allocate memory to format program command line");
! return NULL;
}
if (vsnprintf(cmd, maxargs, fmt, args) > maxargs) {
! warnx("argument list is too long");
! return NULL;
}
+ return cmd;
+ }
+
+ char *
+ system_cmd(const char *fmt, ...)
+ {
+ va_list args;
+ char *cmd;
+
+ va_start(args, fmt);
+ cmd = va_system_cmd(fmt,args);
+ va_end(args);
+ return cmd;
+ }
+
+ /*
+ * Unusual system() substitute. Accepts format string and args,
+ * builds and executes command. Returns exit code.
+ */
+ int
+ vsystem(const char *fmt, ...)
+ {
+ va_list args;
+ char *cmd;
+ int ret;
+
+ va_start(args, fmt);
+ cmd = va_system_cmd(fmt,args);
+ va_end(args);
+ if(cmd == NULL) return 1;
#ifdef DEBUG
printf("Executing %s\n", cmd);
#endif
ret = system(cmd);
free(cmd);
return ret;
}
***************
*** 63,69 ****
{
FILE *fp;
char *cmd, *rp;
- int maxargs;
va_list args;
rp = malloc(MAXPATHLEN);
--- 87,92 ----
***************
*** 71,94 ****
warnx("vpipe can't alloc buffer space");
return NULL;
}
! maxargs = sysconf(_SC_ARG_MAX);
! maxargs -= 32; /* some slop for the sh -c */
! cmd = alloca(maxargs);
! if (!cmd) {
! warnx("vpipe can't alloc arg space");
! return NULL;
! }
- va_start(args, fmt);
- if (vsnprintf(cmd, maxargs, fmt, args) > maxargs) {
- warnx("vsystem args are too long");
- return NULL;
- }
#ifdef DEBUG
fprintf(stderr, "Executing %s\n", cmd);
#endif
fflush(NULL);
fp = popen(cmd, "r");
if (fp == NULL) {
warnx("popen() failed");
return NULL;
--- 94,110 ----
warnx("vpipe can't alloc buffer space");
return NULL;
}
! va_start(args,fmt);
! cmd = va_system_cmd(fmt,args);
! va_end(args);
! if(cmd == NULL) return NULL;
#ifdef DEBUG
fprintf(stderr, "Executing %s\n", cmd);
#endif
fflush(NULL);
fp = popen(cmd, "r");
+ free(cmd);
if (fp == NULL) {
warnx("popen() failed");
return NULL;
***************
*** 97,103 ****
#ifdef DEBUG
fprintf(stderr, "Returned %s\n", rp);
#endif
- va_end(args);
if (pclose(fp) || (strlen(rp) == 0)) {
free(rp);
return NULL;
--- 113,118 ----
Index: lib/file.c
===================================================================
RCS file: /usr/src/cvs/src/usr.sbin/pkg_install/lib/file.c,v
retrieving revision 1.64
diff -c -r1.64 file.c
*** lib/file.c 6 Jan 2003 07:39:02 -0000 1.64
--- lib/file.c 29 Mar 2003 06:12:55 -0000
***************
*** 27,32 ****
--- 27,36 ----
#include <time.h>
#include <sys/wait.h>
+ static int file_is_gzip(const unsigned char *, size_t);
+ static int file_is_bzip2(const unsigned char *, size_t);
+
+
/* Quick check to see if a file exists */
Boolean
fexists(const char *fname)
***************
*** 42,48 ****
isdir(const char *fname)
{
struct stat sb;
!
if (lstat(fname, &sb) != FAIL && S_ISDIR(sb.st_mode))
return TRUE;
else if (lstat(strconcat(fname, "/."), &sb) != FAIL && S_ISDIR(sb.st_mode))
--- 46,52 ----
isdir(const char *fname)
{
struct stat sb;
!
if (lstat(fname, &sb) != FAIL && S_ISDIR(sb.st_mode))
return TRUE;
else if (lstat(strconcat(fname, "/."), &sb) != FAIL && S_ISDIR(sb.st_mode))
***************
*** 328,362 ****
int
unpack(const char *pkg, const char *flist)
{
! char args[10], suff[80], *cp;
! args[0] = '\0';
! /*
! * Figure out by a crude heuristic whether this or not this is probably
! * compressed and whichever compression utility was used (gzip or bzip2).
! */
if (strcmp(pkg, "-")) {
! cp = strrchr(pkg, '.');
! if (cp) {
! strcpy(suff, cp + 1);
! if (strchr(suff, 'z') || strchr(suff, 'Z')) {
! if (strchr(suff, 'b'))
! strcpy(args, "-j");
! else
! strcpy(args, "-z");
! }
! }
}
! else
! /* XXX: need to handle .tgz also */
! strcpy(args, "-j");
! strcat(args, " -xpf");
! if (vsystem("tar %s '%s' %s", args, pkg, flist ? flist : "")) {
warnx("tar extract of %s failed!", pkg);
return 1;
}
return 0;
}
/*
* Using fmt, replace all instances of:
--- 332,411 ----
int
unpack(const char *pkg, const char *flist)
{
! char *cmd;
! char *compression;
! FILE *pkg_file;
! FILE *out_pipe;
! unsigned char *buff;
! size_t buff_allocation;
! size_t buff_size;
! buff_allocation = 8*1024;
!
! buff = (unsigned char *)malloc(buff_allocation);
! /* Determine compression by inspecting file signature */
if (strcmp(pkg, "-")) {
! pkg_file = fopen(pkg,"r");
! } else {
! pkg_file = stdin;
}
! if(pkg_file == NULL) {
! warnx("couldn't open %s",pkg);
! return 1;
! }
!
! /* Select appropriate compression argument for tar by
! * inspecting file signature */
! buff_size = fread(buff,1,buff_allocation,pkg_file);
! if(file_is_gzip(buff,buff_size)) {
! compression = "z";
! } else if(file_is_bzip2(buff,buff_size)) {
! compression = "j";
! } else {
! compression = "";
! }
!
! cmd = system_cmd("tar -xp%sf - %s",compression,flist ? flist : "");
! if(cmd == NULL) return 1;
!
! /* Cat entire file through tar */
! #ifdef DEBUG
! printf("Piping package '%s' to cmd '%s'\n", pkg,cmd);
! #endif
! out_pipe = popen(cmd,"w");
! free(cmd);
!
! if(out_pipe == NULL) return 1;
! while(buff_size > 0) {
! if(buff_size < fwrite(buff,1,buff_size,out_pipe))
! break;
! buff_size = fread(buff,1,buff_allocation,pkg_file);
! }
! if(pclose(out_pipe)) {
warnx("tar extract of %s failed!", pkg);
return 1;
}
return 0;
}
+
+ /*
+ * Returns 1 if buffer holds initial bytes of a gzipped file
+ */
+ static int
+ file_is_gzip(const unsigned char *head, size_t s) {
+ if(s < 2) return 0;
+ return (head[0]==0037 && head[1]==0213);
+ }
+
+ /*
+ * Returns 1 if buffer holds initial bytes of a bzip2-ed file
+ */
+ static int
+ file_is_bzip2(const unsigned char *head, size_t s) {
+ if(s < 3) return 0;
+ return (head[0]=='B' && head[1]=='Z' && head[2]=='h');
+ }
+
/*
* Using fmt, replace all instances of:
Index: lib/lib.h
===================================================================
RCS file: /usr/src/cvs/src/usr.sbin/pkg_install/lib/lib.h,v
retrieving revision 1.47
diff -c -r1.47 lib.h
*** lib/lib.h 6 Jan 2003 07:39:02 -0000 1.47
--- lib/lib.h 29 Mar 2003 06:00:36 -0000
***************
*** 137,142 ****
--- 137,143 ----
/* Prototypes */
/* Misc */
int vsystem(const char *, ...);
+ char * system_cmd(const char *, ...);
char *vpipe(const char *, ...);
void cleanup(int);
char *make_playpen(char *, off_t);
--------------070001080502070007020104--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?3E85418F.8010201>