Date: Wed, 15 Apr 2026 13:38:23 +0000 From: Vladimir Druzenko <vvd@FreeBSD.org> To: ports-committers@FreeBSD.org, dev-commits-ports-all@FreeBSD.org, dev-commits-ports-main@FreeBSD.org Cc: Generic Rikka <rikka.goering@outlook.de> Subject: git: 5f283cddd7b7 - main - sysutils/mpifileutils: New port: MPI parallel file utilities for high scalability on large filesystems Message-ID: <69df94cf.42f6a.5b62bb46@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by vvd: URL: https://cgit.FreeBSD.org/ports/commit/?id=5f283cddd7b7c17cd59a5a887bf4ae74566ccbd1 commit 5f283cddd7b7c17cd59a5a887bf4ae74566ccbd1 Author: Generic Rikka <rikka.goering@outlook.de> AuthorDate: 2026-04-15 13:23:56 +0000 Commit: Vladimir Druzenko <vvd@FreeBSD.org> CommitDate: 2026-04-15 13:37:45 +0000 sysutils/mpifileutils: New port: MPI parallel file utilities for high scalability on large filesystems FileUtils provides a library (libmfu) and a suite of MPI-parallel file utilities such as dcp (parallel copy), dcmp (compare), ddup (duplicate finder), dfind, dtar/dbz2, dchmod, and more. Designed for high scalability on large filesystems. https://hpc.github.io/mpifileutils/ https://github.com/hpc/mpifileutils/ Pull request to upstream repo with local patches: https://github.com/hpc/mpifileutils/pull/664 PR: 291679 Sponsored by: UNIS Labs Co-authored-by: Vladimir Druzenko <vvd@FreeBSD.org> --- sysutils/Makefile | 1 + sysutils/mpifileutils/Makefile | 39 ++++++++++++ sysutils/mpifileutils/distinfo | 3 + sysutils/mpifileutils/files/patch-CMakeLists.txt | 20 ++++++ .../files/patch-mpifileutils_src_common_mfu__bz2.c | 15 +++++ .../files/patch-mpifileutils_src_common_mfu__bz2.h | 10 +++ ...atch-mpifileutils_src_common_mfu__bz2__static.c | 24 +++++++ ...tils_src_common_mfu__compress__bz2__libcircle.c | 28 +++++++++ ...ls_src_common_mfu__decompress__bz2__libcircle.c | 12 ++++ .../patch-mpifileutils_src_common_mfu__flist.c | 44 +++++++++++++ .../patch-mpifileutils_src_common_mfu__flist.h | 11 ++++ ...h-mpifileutils_src_common_mfu__flist__archive.c | 32 ++++++++++ ...atch-mpifileutils_src_common_mfu__flist__copy.c | 56 +++++++++++++++++ ...atch-mpifileutils_src_common_mfu__flist__walk.c | 41 ++++++++++++ .../files/patch-mpifileutils_src_common_mfu__io.c | 15 +++++ .../files/patch-mpifileutils_src_common_mfu__io.h | 20 ++++++ ...atch-mpifileutils_src_common_mfu__param__path.c | 40 ++++++++++++ .../patch-mpifileutils_src_common_mfu__util.c | 46 ++++++++++++++ .../patch-mpifileutils_src_common_mfu__util.h | 13 ++++ .../files/patch-mpifileutils_src_dbcast_dbcast.c | 73 ++++++++++++++++++++++ .../files/patch-mpifileutils_src_dbz2_dbz2.c | 12 ++++ .../files/patch-mpifileutils_src_dcmp_dcmp.c | 15 +++++ .../files/patch-mpifileutils_src_dcp1_cleanup.c | 13 ++++ .../files/patch-mpifileutils_src_dcp1_common.h | 11 ++++ .../files/patch-mpifileutils_src_dsync_dsync.c | 22 +++++++ sysutils/mpifileutils/pkg-descr | 4 ++ sysutils/mpifileutils/pkg-plist | 43 +++++++++++++ 27 files changed, 663 insertions(+) diff --git a/sysutils/Makefile b/sysutils/Makefile index 047229215e8a..1238a20ffe8c 100644 --- a/sysutils/Makefile +++ b/sysutils/Makefile @@ -755,6 +755,7 @@ SUBDIR += moreutils SUBDIR += most SUBDIR += mountsmb2 + SUBDIR += mpifileutils SUBDIR += mping SUBDIR += mprocs SUBDIR += mptd diff --git a/sysutils/mpifileutils/Makefile b/sysutils/mpifileutils/Makefile new file mode 100644 index 000000000000..d9bca22b6b88 --- /dev/null +++ b/sysutils/mpifileutils/Makefile @@ -0,0 +1,39 @@ +PORTNAME= mpifileutils +DISTVERSIONPREFIX= v +DISTVERSION= 0.12 +CATEGORIES= sysutils parallel +MASTER_SITES= https://github.com/hpc/${PORTNAME}/releases/download/v${DISTVERSION}/ + +MAINTAINER= rikka.goering@outlook.de +COMMENT= MPI-based parallel file utilities (dcp, dcmp, ddup, dtar, etc.) +WWW= https://hpc.github.io/mpifileutils/ + +LICENSE= BSD3CLAUSE +LICENSE_FILE= ${WRKSRC}/${PORTNAME}/LICENSE + +LIB_DEPENDS= libcircle.so:devel/libcircle \ + libdtcmp.so:devel/dtcmp + +USES= cmake localbase:ldflags mpi:openmpi pkgconfig ssl tar:tgz +USE_LDCONFIG= yes + +CMAKE_ARGS= -DCMAKE_C_COMPILER=${MPICC} \ + -DCMAKE_CXX_COMPILER=${MPICXX} +CMAKE_OFF= ENABLE_GPFS ENABLE_LUSTRE ENABLE_XATTRS + +LDFLAGS+= -lcircle -ldtcmp + +EXTRACT_AFTER_ARGS= --exclude dtcmp \ + --exclude libcircle \ + --exclude lwgrp \ + --no-same-owner --no-same-permissions + +PORTDOCS= AUTHORS CONTRIBUTING.md DAOS-Support.md NOTICE README.md + +OPTIONS_DEFINE= DOCS + +do-install-DOCS-on: + @${MKDIR} ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${PORTDOCS:S|^|${WRKSRC}/${PORTNAME}/|} ${STAGEDIR}${DOCSDIR} + +.include <bsd.port.mk> diff --git a/sysutils/mpifileutils/distinfo b/sysutils/mpifileutils/distinfo new file mode 100644 index 000000000000..66d05253271e --- /dev/null +++ b/sysutils/mpifileutils/distinfo @@ -0,0 +1,3 @@ +TIMESTAMP = 1775666666 +SHA256 (mpifileutils-v0.12.tgz) = 7a5ed469be579bc0e9480febc127991467bc71aa37ded083265b859245300206 +SIZE (mpifileutils-v0.12.tgz) = 580255 diff --git a/sysutils/mpifileutils/files/patch-CMakeLists.txt b/sysutils/mpifileutils/files/patch-CMakeLists.txt new file mode 100644 index 000000000000..e5f1b0ed453b --- /dev/null +++ b/sysutils/mpifileutils/files/patch-CMakeLists.txt @@ -0,0 +1,20 @@ +--- CMakeLists.txt.orig 2025-04-17 19:28:45 UTC ++++ CMakeLists.txt +@@ -298,9 +298,6 @@ LIST(APPEND libmfu_srcs + mpifileutils/src/common/mfu_progress.c + mpifileutils/src/common/mfu_util.c + mpifileutils/src/common/strmap.c +- ${lwgrp_srcs} +- ${dtcmp_srcs} +- ${libcircle_srcs} + ) + IF(ENABLE_LIBARCHIVE) + LIST(APPEND libmfu_srcs +@@ -328,7 +325,3 @@ INSTALL(TARGETS mfu-static DESTINATION ${CMAKE_INSTALL + + # some projects require a "make install" command to work, + # so define at least a basic INSTALL function +-INSTALL(FILES lwgrp/README lwgrp/LICENSE.TXT DESTINATION share/lwgrp) +-INSTALL(FILES dtcmp/README.md dtcmp/LICENSE.TXT DESTINATION share/dtcmp) +-INSTALL(FILES libcircle/COPYING DESTINATION share/libcircle) +-INSTALL(FILES mpifileutils/LICENSE mpifileutils/NOTICE DESTINATION share/mpifileutils) diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2.c new file mode 100644 index 000000000000..31b968d1550b --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2.c @@ -0,0 +1,15 @@ +--- mpifileutils/src/common/mfu_bz2.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_bz2.c +@@ -6,7 +6,12 @@ + #include <errno.h> + + /* for statfs */ ++#if defined(__linux__) + #include <sys/vfs.h> ++#elif defined(__FreeBSD__) ++#include <sys/param.h> ++#include <sys/mount.h> ++#endif + + /* for LL_SUPER_MAGIC */ + #if LUSTRE_SUPPORT diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2.h b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2.h new file mode 100644 index 000000000000..098848f25249 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2.h @@ -0,0 +1,10 @@ +--- mpifileutils/src/common/mfu_bz2.h.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_bz2.h +@@ -1,6 +1,7 @@ + #ifndef MFU_BZ2_H + #define MFU_BZ2_H + ++int mfu_compress_bz2_static(const char* src_name, const char* dst_name, int b_size); + int mfu_compress_bz2(const char* src_name, const char* dst_name, int b_size); + int mfu_decompress_bz2(const char* src_name, const char* dst_name); + diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2__static.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2__static.c new file mode 100644 index 000000000000..59c93a8a4242 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__bz2__static.c @@ -0,0 +1,24 @@ +--- mpifileutils/src/common/mfu_bz2_static.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_bz2_static.c +@@ -5,7 +5,9 @@ + #include <unistd.h> + #include <stdio.h> + #include <stdlib.h> ++#if defined(__linux__) + #include <sys/sysinfo.h> ++#endif + #include <string.h> + #include <sys/time.h> + #include <sys/resource.h> +@@ -22,6 +24,11 @@ + #include "libcircle.h" + #include "mfu.h" + #include "mfu_bz2.h" ++ ++/* FreeBSD does not define O_LARGEFILE; large file support is implicit. */ ++#ifndef O_LARGEFILE ++#define O_LARGEFILE 0 ++#endif + + #define FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH) + diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__compress__bz2__libcircle.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__compress__bz2__libcircle.c new file mode 100644 index 000000000000..9bbcbc785c05 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__compress__bz2__libcircle.c @@ -0,0 +1,28 @@ +--- mpifileutils/src/common/mfu_compress_bz2_libcircle.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_compress_bz2_libcircle.c +@@ -1,3 +1,20 @@ ++#if !defined(__linux__) ++/* ++ * Non-Linux fallback: ++ * The libcircle-based implementation relies on sysinfo(2) / <sys/sysinfo.h>. ++ * FreeBSD doesn't provide that API, so fall back to the portable static path. ++ */ ++#include "mfu.h" ++#include "mfu_bz2.h" ++ ++int mfu_compress_bz2_libcircle(const char* src, const char* dst, int b_size, ssize_t opts_memory) ++{ ++ (void) opts_memory; ++ return mfu_compress_bz2_static(src, dst, b_size); ++} ++ ++#else /* __linux__ */ ++ + #ifndef _GNU_SOURCE + #define _GNU_SOURCE + #endif +@@ -531,3 +548,4 @@ int mfu_compress_bz2_libcircle(const char* src, const + + return rc; + } ++#endif /* __linux__ */ diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__decompress__bz2__libcircle.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__decompress__bz2__libcircle.c new file mode 100644 index 000000000000..3b7a2342f7b5 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__decompress__bz2__libcircle.c @@ -0,0 +1,12 @@ +--- mpifileutils/src/common/mfu_decompress_bz2_libcircle.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_decompress_bz2_libcircle.c +@@ -5,7 +5,9 @@ + #include <unistd.h> + #include <stdio.h> + #include <stdlib.h> ++#if defined(__linux__) + #include <sys/sysinfo.h> ++#endif + #include <string.h> + #include <sys/time.h> + #include <sys/resource.h> diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist.c new file mode 100644 index 000000000000..44b350be7c0d --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist.c @@ -0,0 +1,44 @@ +--- mpifileutils/src/common/mfu_flist.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_flist.c +@@ -35,8 +35,13 @@ + #include <sys/ioctl.h> + #include <sys/param.h> + ++/* Linux-specific file flag ioctls live in <linux/fs.h>; ++ * not available (and not used) on FreeBSD. ++ */ ++#ifdef __linux__ + #include <linux/fs.h> + #include <linux/fiemap.h> ++#endif + + #include <libgen.h> /* dirname */ + #include "libcircle.h" +@@ -845,7 +850,7 @@ uint64_t mfu_flist_file_get_perm(mfu_flist bflist, uin + return mode & (S_IRWXU | S_IRWXG | S_IRWXO); + } + +-#if DCOPY_USE_XATTRS ++#if defined(__linux__) && DCOPY_USE_XATTRS + void *mfu_flist_file_get_acl(mfu_flist bflist, uint64_t idx, ssize_t *acl_size, char *type) + { + flist_t* flist = (flist_t*) bflist; +@@ -886,7 +891,18 @@ void *mfu_flist_file_get_acl(mfu_flist bflist, uint64_ + + return val; + } ++#else ++void *mfu_flist_file_get_acl(mfu_flist bflist, uint64_t idx, ssize_t *acl_size, char *type) ++{ ++ (void)bflist; ++ (void)idx; ++ (void)type; ++ if (acl_size != NULL) ++ *acl_size = 0; ++ return NULL; ++} + #endif ++ + + uint64_t mfu_flist_file_get_uid(mfu_flist bflist, uint64_t idx) + { diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist.h b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist.h new file mode 100644 index 000000000000..8ad223af3c18 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist.h @@ -0,0 +1,11 @@ +--- mpifileutils/src/common/mfu_flist.h.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_flist.h +@@ -44,7 +44,7 @@ extern "C" { + #include <stdbool.h> + #include "mpi.h" + +-#if DCOPY_USE_XATTRS ++#if defined(__linux__) && DCOPY_USE_XATTRS + #include <sys/xattr.h> + /* + * Newer versions of attr deprecated attr/xattr.h which defines ENOATTR as a diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__archive.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__archive.c new file mode 100644 index 000000000000..b22fdf92d9ed --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__archive.c @@ -0,0 +1,32 @@ +--- mpifileutils/src/common/mfu_flist_archive.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_flist_archive.c +@@ -27,6 +27,10 @@ + #include <string.h> + #include <getopt.h> + ++#ifndef O_LARGEFILE ++#define O_LARGEFILE 0 ++#endif ++ + /* gettimeofday */ + #include <sys/time.h> + +@@ -5255,6 +5259,7 @@ static int extract_xattrs( + r = archive_entry_xattr_next(entry, &xname, &xval, &xsize); + if (r == ARCHIVE_OK) { + /* successfully extracted xattr, now try to set it */ ++#if defined(__linux__) + int set_rc = setxattr(path, xname, xval, xsize, 0); + if (set_rc == -1) { + MFU_LOG(MFU_LOG_ERR, "failed to setxattr '%s' on '%s' errno=%d %s", +@@ -5263,6 +5268,10 @@ static int extract_xattrs( + rc = MFU_FAILURE; + /* don't break in case other xattrs work */ + } ++#else ++ /* FreeBSD: implement Linux setxattr() equivalent using extattr(2) */ ++ (void)path; (void)xname; (void)xval; (void)xsize; ++#endif + } else { + /* failed to read xattr */ + MFU_LOG(MFU_LOG_ERR, "failed to extract xattr for '%s' of entry %llu at offset %llu", diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__copy.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__copy.c new file mode 100644 index 000000000000..142e69fd490e --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__copy.c @@ -0,0 +1,56 @@ +--- mpifileutils/src/common/mfu_flist_copy.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_flist_copy.c +@@ -39,8 +39,10 @@ + #include <sys/ioctl.h> + #include <sys/param.h> + ++#ifdef __linux__ + #include <linux/fs.h> + #include <linux/fiemap.h> ++#endif + + /* define PRI64 */ + #include <inttypes.h> +@@ -69,8 +71,10 @@ + #endif + + #ifdef HPSS_SUPPORT ++#ifdef __linux__ + #include <linux/hpssfs.h> + #endif ++#endif + + /**************************************** + * Define types +@@ -1864,6 +1868,7 @@ static int mfu_copy_file_normal( + return 0; + } + ++#ifdef __linux__ + static int mfu_copy_file_fiemap( + const char* src, + const char* dest, +@@ -2053,6 +2058,7 @@ fail_normal_copy: + fail_normal_copy: + return -1; + } ++#endif + + static int mfu_copy_file( + const char* src, +@@ -2084,6 +2090,7 @@ static int mfu_copy_file( + return -1; + } + ++#ifdef __linux__ + if (copy_opts->sparse) { + bool normal_copy_required; + ret = mfu_copy_file_fiemap(src, dest, offset, length, file_size, +@@ -2093,6 +2100,7 @@ static int mfu_copy_file( + return ret; + } + } ++#endif + + ret = mfu_copy_file_normal(src, dest, offset, length, file_size, + copy_opts, mfu_src_file, mfu_dst_file); diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__walk.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__walk.c new file mode 100644 index 000000000000..352120e731b8 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__flist__walk.c @@ -0,0 +1,41 @@ +--- mpifileutils/src/common/mfu_flist_walk.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_flist_walk.c +@@ -186,8 +186,10 @@ static void walk_getdents_process_dir(const char* dir, + int flags = O_RDONLY | O_DIRECTORY; + char buf[BUF_SIZE]; + ++#ifdef __linux__ + if (NO_ATIME) + flags |= O_NOATIME; ++#endif + + /* TODO: may need to try these functions multiple times */ + mfu_file_t* mfu_file = *CURRENT_PFILE; +@@ -205,7 +207,17 @@ static void walk_getdents_process_dir(const char* dir, + /* Read all directory entries */ + while (1) { + /* execute system call to get block of directory entries */ +- int nread = syscall(SYS_getdents, mfu_file->fd, buf, (int) BUF_SIZE); ++ int nread; ++#ifdef __linux__ ++ /* On Linux, call the raw getdents syscall as before */ ++ nread = syscall(SYS_getdents, mfu_file->fd, buf, (int) BUF_SIZE); ++#else ++ /* ++ * On FreeBSD and other non-Linux systems, use the getdents(2) ++ * libc wrapper instead of the Linux-specific SYS_getdents. ++ */ ++ nread = getdents(mfu_file->fd, (char *)buf, (size_t)BUF_SIZE); ++#endif + if (nread == -1) { + MFU_LOG(MFU_LOG_ERR, "syscall to getdents failed when reading `%s' (errno=%d %s)", dir, errno, strerror(errno)); + WALK_RESULT = -1; +@@ -767,7 +779,7 @@ void mfu_flist_stat( + /* check whether we should skip this item */ + if (skip_fn != NULL && skip_fn(name, skip_args)) { + /* skip this file, don't include it in new list */ +- MFU_LOG(MFU_LOG_INFO, "skip %s"); ++ MFU_LOG(MFU_LOG_INFO, "skip %s", name); + continue; + } + diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__io.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__io.c new file mode 100644 index 000000000000..b4616a833483 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__io.c @@ -0,0 +1,15 @@ +--- mpifileutils/src/common/mfu_io.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_io.c +@@ -21,6 +21,12 @@ + #include "mfu.h" + #include "mfu_errors.h" + ++/* FreeBSD does not provide lstat64, only lstat. Map it accordingly. */ ++#ifdef __FreeBSD__ ++#undef lstat64 ++#define lstat64(path, buf) lstat((path), (struct stat *)(buf)) ++#endif ++ + #define MFU_IO_TRIES (5) + #define MFU_IO_USLEEP (100) + diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__io.h b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__io.h new file mode 100644 index 000000000000..e66be9c943f3 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__io.h @@ -0,0 +1,20 @@ +--- mpifileutils/src/common/mfu_io.h.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_io.h +@@ -42,12 +42,15 @@ extern "C" { + + #include "mfu_param_path.h" + ++#ifdef __FreeBSD__ ++#define stat64 stat ++#define lstat64 lstat ++#endif ++ + /* Intent is to wrap all POSIX I/O routines used by mfu tools. May + * abort on fatal conditions to avoid checking condition at every call. + * May also automatically retry on things like EINTR. */ + +-/* TODO: fix this */ +-/* do this to avoid warning about undefined stat64 struct */ + struct stat64; + + /***************************** diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__param__path.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__param__path.c new file mode 100644 index 000000000000..21640f757a64 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__param__path.c @@ -0,0 +1,40 @@ +--- mpifileutils/src/common/mfu_param_path.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_param_path.c +@@ -1,5 +1,23 @@ + #define _GNU_SOURCE + ++/* Ensure POSIX faccessat(2) AT_* constants are visible on FreeBSD */ ++#ifndef _POSIX_C_SOURCE ++#define _POSIX_C_SOURCE 200809L ++#endif ++ ++#include <sys/types.h> ++#include <sys/stat.h> ++#include <fcntl.h> ++#include <unistd.h> ++ ++/* Fallbacks (in case headers hide these behind visibility macros) */ ++#ifndef AT_FDCWD ++#define AT_FDCWD (-100) ++#endif ++#ifndef AT_SYMLINK_NOFOLLOW ++#define AT_SYMLINK_NOFOLLOW 0 ++#endif ++ + #include "mfu.h" + #include "mpi.h" + +@@ -8,6 +26,13 @@ + #include <string.h> + #include <stdarg.h> + #include <errno.h> ++ ++#include <sys/types.h> ++#include <sys/stat.h> ++ ++#ifndef S_ISLNK ++#define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK) ++#endif + + #include <fcntl.h> + #include <unistd.h> diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__util.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__util.c new file mode 100644 index 000000000000..b3c5d8ffd900 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__util.c @@ -0,0 +1,46 @@ +--- mpifileutils/src/common/mfu_util.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_util.c +@@ -1,6 +1,11 @@ + /* to get nsec fields in stat structure */ + #define _GNU_SOURCE + ++#if defined(__FreeBSD__) ++#include <sys/types.h> ++#include <sys/param.h> ++#endif ++ + #include "mfu.h" + #include "mpi.h" + #include "dtcmp.h" +@@ -14,12 +19,21 @@ + #include <errno.h> + #include <limits.h> + +-#include <sys/types.h> + #include <sys/stat.h> ++#include <fcntl.h> + #include <unistd.h> + ++#if defined(__linux__) + #include <sys/vfs.h> ++#elif defined(__FreeBSD__) ++#include <sys/param.h> ++#include <sys/mount.h> ++#endif + ++#ifndef S_ISLNK ++#define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK) ++#endif ++ + #ifndef ULLONG_MAX + #define ULLONG_MAX (__LONG_LONG_MAX__ * 2UL + 1UL) + #endif +@@ -939,7 +953,7 @@ int mfu_compare_contents( + /* check for write error */ + if (bytes_written < 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to write `%s' at offset %llx (errno=%d %s)", +- dst_name, (unsigned long long)off + n, strerror(errno)); ++ dst_name, (unsigned long long)off + n, errno, strerror(errno)); + rc = -1; + break; + } diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__util.h b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__util.h new file mode 100644 index 000000000000..2b2c06a1ed6e --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_common_mfu__util.h @@ -0,0 +1,13 @@ +--- mpifileutils/src/common/mfu_util.h.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/common/mfu_util.h +@@ -9,6 +9,10 @@ extern "C" { + #ifndef MFU_UTIL_H + #define MFU_UTIL_H + ++#ifndef O_NOATIME ++#define O_NOATIME 0 ++#endif ++ + #include <stdlib.h> + #include <stdarg.h> + #include <stdint.h> diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_dbcast_dbcast.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_dbcast_dbcast.c new file mode 100644 index 000000000000..6f83c35419e2 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_dbcast_dbcast.c @@ -0,0 +1,73 @@ +--- mpifileutils/src/dbcast/dbcast.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/dbcast/dbcast.c +@@ -16,7 +16,12 @@ + #include <string.h> + #include <getopt.h> + ++#ifdef __linux__ + #include <sys/vfs.h> ++#elif defined(__FreeBSD__) ++#include <sys/param.h> ++#include <sys/mount.h> ++#endif + + // mmap and friends for shared memory + #include <sys/mman.h> +@@ -32,6 +37,14 @@ + * and send data to each other. The writer flow controls these worker + * processes with messages. */ + ++#ifndef HOST_NAME_MAX ++# ifdef MAXHOSTNAMELEN ++# define HOST_NAME_MAX MAXHOSTNAMELEN ++# else ++# define HOST_NAME_MAX 256 ++# endif ++#endif ++ + #define GCS_SUCCESS (0) + + static int gcs_shm_file_key = MPI_KEYVAL_INVALID; +@@ -803,7 +816,7 @@ int main (int argc, char *argv[]) + /* check whether we have space to write the file */ + if (file_size > free_size) { + /* not enough space for file */ +- MFU_LOG(MFU_LOG_ERR, "Insufficient space for file `%s` filesize=%llu free=%llu", out_file_path, file_size, free_size); ++ MFU_LOG(MFU_LOG_ERR, "Insufficient space for file `%s` filesize=%llu free=%llu", out_file_path, (unsigned long long)file_size, (unsigned long long)free_size); + write_error = 1; + } + } +@@ -1118,14 +1131,14 @@ if (node_rank == 0) { + * a file by the same name but of different size than a previous copy */ + errno = 0; + if (mfu_truncate(out_file_path, (off_t) file_size) != 0) { +- MFU_LOG(MFU_LOG_ERR, "Failed to truncate file `%s`", out_file_path, strerror(errno)); ++ MFU_LOG(MFU_LOG_ERR, "Failed to truncate file `%s`: %s", out_file_path, strerror(errno)); + write_error = 1; + } + + /* have every writer update file mode */ + errno = 0; + if (mfu_chmod(out_file_path, (mode_t) mode) != 0) { +- MFU_LOG(MFU_LOG_ERR, "Failed to chmod file `%s`", out_file_path, strerror(errno)); ++ MFU_LOG(MFU_LOG_ERR, "Failed to chmod file `%s`: %s", out_file_path, strerror(errno)); + write_error = 1; + } + +@@ -1133,14 +1146,14 @@ if (node_rank == 0) { + if (write_error) { + errno = 0; + if (mfu_unlink(out_file_path) != 0) { +- MFU_LOG(MFU_LOG_ERR, "Failed to unlink file `%s`", out_file_path, strerror(errno)); ++ MFU_LOG(MFU_LOG_ERR, "Failed to unlink file `%s`: %s", out_file_path, strerror(errno)); + } + } + } else { + /* readers close input file */ + errno = 0; + if (mfu_close(in_file_path, in_file) != 0) { +- MFU_LOG(MFU_LOG_ERR, "Failed to close file `%s`", in_file_path, strerror(errno)); ++ MFU_LOG(MFU_LOG_ERR, "Failed to close file `%s`: %s", in_file_path, strerror(errno)); + } + } + diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_dbz2_dbz2.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_dbz2_dbz2.c new file mode 100644 index 000000000000..9f1683d6e0d3 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_dbz2_dbz2.c @@ -0,0 +1,12 @@ +--- mpifileutils/src/dbz2/dbz2.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/dbz2/dbz2.c +@@ -1,7 +1,9 @@ + #include <unistd.h> + #include <stdio.h> + #include <stdlib.h> ++#if defined(__linux__) + #include <sys/sysinfo.h> ++#endif + #include <string.h> + #include <sys/time.h> + #include <sys/resource.h> diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_dcmp_dcmp.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_dcmp_dcmp.c new file mode 100644 index 000000000000..a173f92c5b92 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_dcmp_dcmp.c @@ -0,0 +1,15 @@ +--- mpifileutils/src/dcmp/dcmp.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/dcmp/dcmp.c +@@ -4,7 +4,12 @@ + #include <stdlib.h> + #include <mpi.h> + #include <libcircle.h> ++#ifdef __linux__ + #include <linux/limits.h> ++#elif defined(__FreeBSD__) ++#include <limits.h> ++#include <sys/param.h> ++#endif + #include <libgen.h> + #include <errno.h> + #include <dtcmp.h> diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_dcp1_cleanup.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_dcp1_cleanup.c new file mode 100644 index 000000000000..c477df8d054e --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_dcp1_cleanup.c @@ -0,0 +1,13 @@ +--- mpifileutils/src/dcp1/cleanup.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/dcp1/cleanup.c +@@ -51,8 +51,8 @@ static void DCOPY_truncate_file(DCOPY_operation_t* op, + * Try the recursive file before file-to-file. The cast below requires us + * to have a maximum file_size of 2^63, not 2^64. + */ +- if(truncate64(dest_path_recursive, op->file_size) < 0) { +- if(truncate64(dest_path_file_to_file, op->file_size) < 0) { ++ if(mfu_truncate(dest_path_recursive, op->file_size) < 0) { ++ if(mfu_truncate(dest_path_file_to_file, op->file_size) < 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to truncate destination file: %s (errno=%d %s)", + dest_path_recursive, errno, strerror(errno)); + diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_dcp1_common.h b/sysutils/mpifileutils/files/patch-mpifileutils_src_dcp1_common.h new file mode 100644 index 000000000000..abef2e0d1bd6 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_dcp1_common.h @@ -0,0 +1,11 @@ +--- mpifileutils/src/dcp1/common.h.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/dcp1/common.h +@@ -44,7 +44,7 @@ + #include <unistd.h> + #include <utime.h> + +-#if DCOPY_USE_XATTRS ++#if defined(__linux__) && DCOPY_USE_XATTRS + #include <sys/xattr.h> + /* + * Newer versions of attr deprecated attr/xattr.h which defines ENOATTR as a diff --git a/sysutils/mpifileutils/files/patch-mpifileutils_src_dsync_dsync.c b/sysutils/mpifileutils/files/patch-mpifileutils_src_dsync_dsync.c new file mode 100644 index 000000000000..95448ee51f44 --- /dev/null +++ b/sysutils/mpifileutils/files/patch-mpifileutils_src_dsync_dsync.c @@ -0,0 +1,22 @@ +--- mpifileutils/src/dsync/dsync.c.orig 2025-02-19 22:20:52 UTC ++++ mpifileutils/src/dsync/dsync.c +@@ -24,7 +24,7 @@ + #include <stdlib.h> + #include <mpi.h> + #include <libcircle.h> +-#include <linux/limits.h> ++#include <limits.h> + #include <libgen.h> + #include <errno.h> + #include <dtcmp.h> +@@ -32,6 +32,10 @@ + #define _XOPEN_SOURCE 600 + #include <fcntl.h> + #include <string.h> ++ ++#ifndef PATH_MAX ++#define PATH_MAX 4096 ++#endif + + /* for bool type, true/false macros */ + #include <stdbool.h> diff --git a/sysutils/mpifileutils/pkg-descr b/sysutils/mpifileutils/pkg-descr new file mode 100644 index 000000000000..5c848562a0db --- /dev/null +++ b/sysutils/mpifileutils/pkg-descr @@ -0,0 +1,4 @@ +mpiFileUtils provides a library (libmfu) and a suite of MPI-parallel file +utilities such as dcp (parallel copy), dcmp (compare), ddup (duplicate finder), +dfind, dtar/dbz2, dchmod, and more. Designed for high scalability on large +filesystems. diff --git a/sysutils/mpifileutils/pkg-plist b/sysutils/mpifileutils/pkg-plist new file mode 100644 index 000000000000..e1f40d5fdf31 --- /dev/null +++ b/sysutils/mpifileutils/pkg-plist @@ -0,0 +1,43 @@ +bin/dbcast +bin/dbz2 +bin/dchmod +bin/dcmp +bin/dcp +bin/dcp1 +bin/ddup +bin/dfilemaker +bin/dfind +bin/dreln +bin/drm +bin/dstripe +bin/dsync +bin/dtar +bin/dwalk +include/mfu.h +include/mfu_bz2.h +include/mfu_errors.h +include/mfu_flist.h +include/mfu_flist_internal.h +include/mfu_io.h +include/mfu_param_path.h +include/mfu_path.h +include/mfu_pred.h +include/mfu_progress.h +include/mfu_util.h +lib/libmfu.a +lib/libmfu.so +lib/libmfu.so.4.0.0 +share/man/man1/dbcast.1.gz +share/man/man1/dbz2.1.gz +share/man/man1/dchmod.1.gz +share/man/man1/dcmp.1.gz +share/man/man1/dcp.1.gz +share/man/man1/ddup.1.gz +share/man/man1/dfilemaker.1.gz +share/man/man1/dfind.1.gz +share/man/man1/dreln.1.gz +share/man/man1/drm.1.gz +share/man/man1/dstripe.1.gz +share/man/man1/dsync.1.gz +share/man/man1/dtar.1.gz +share/man/man1/dwalk.1.gzhome | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?69df94cf.42f6a.5b62bb46>