Date: Mon, 16 Sep 2024 20:20:32 GMT From: Robert Clausecker <fuz@FreeBSD.org> To: ports-committers@FreeBSD.org, dev-commits-ports-all@FreeBSD.org, dev-commits-ports-main@FreeBSD.org Subject: git: 833afe4f6fa6 - main - sysutils/zrepl-dsh2dsh: ZFS dataset replication tool (fork with more features) Message-ID: <202409162020.48GKKWv6063081@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by fuz: URL: https://cgit.FreeBSD.org/ports/commit/?id=833afe4f6fa662eb9e9ddbcac028a9b4db21879d commit 833afe4f6fa662eb9e9ddbcac028a9b4db21879d Author: Denis Shaposhnikov <dsh@bamus.cz> AuthorDate: 2024-09-12 10:27:18 +0000 Commit: Robert Clausecker <fuz@FreeBSD.org> CommitDate: 2024-09-16 20:14:23 +0000 sysutils/zrepl-dsh2dsh: ZFS dataset replication tool (fork with more features) Cross-platform ZFS replication tool written in Go that supports feature detection, bookmarks, and other new features. This is a fork with more features like * support of shell patterns for datasets definitions * new log formatters * ability to log into a file * ability to configure command piplines between `zfs send` and `zfs recv` * Icinga/Nagios checks * fast skip "keep all" pruning * snapshots can be named using local time in timestamps * configurable RPC timeout * configurable path to zfs binary * faster replication and so on. WWW: https://github.com/dsh2dsh/zrepl PR: 251576 --- sysutils/Makefile | 1 + sysutils/zrepl-dsh2dsh/Makefile | 47 +++++++++ sysutils/zrepl-dsh2dsh/distinfo | 5 + sysutils/zrepl-dsh2dsh/files/500.zrepl.in | 41 ++++++++ sysutils/zrepl-dsh2dsh/files/newsyslog.conf | 5 + sysutils/zrepl-dsh2dsh/files/pkg-message.in | 35 +++++++ sysutils/zrepl-dsh2dsh/files/zrepl.in | 80 +++++++++++++++ sysutils/zrepl-dsh2dsh/files/zrepl.yml | 148 ++++++++++++++++++++++++++++ sysutils/zrepl-dsh2dsh/pkg-descr | 17 ++++ sysutils/zrepl-dsh2dsh/pkg-plist | 23 +++++ 10 files changed, 402 insertions(+) diff --git a/sysutils/Makefile b/sysutils/Makefile index a90d99f04fc8..c27a1e74b5fe 100644 --- a/sysutils/Makefile +++ b/sysutils/Makefile @@ -1719,6 +1719,7 @@ SUBDIR += zpool-iostat-viz SUBDIR += zrep SUBDIR += zrepl + SUBDIR += zrepl-dsh2dsh SUBDIR += zsd SUBDIR += zsm SUBDIR += ztop diff --git a/sysutils/zrepl-dsh2dsh/Makefile b/sysutils/zrepl-dsh2dsh/Makefile new file mode 100644 index 000000000000..c17df694bca6 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/Makefile @@ -0,0 +1,47 @@ +PORTNAME= zrepl +DISTVERSIONPREFIX= v +DISTVERSION= 0.7.8 +CATEGORIES= sysutils +PKGNAMESUFFIX= -dsh2dsh + +MAINTAINER= dsh@bamus.cz +COMMENT= ZFS dataset replication tool (fork with more features) +WWW= https://github.com/dsh2dsh/${PORTNAME} + +LICENSE= MIT +LICENSE_FILE= ${WRKSRC}/LICENSE + +USES= go:1.22,modules ncurses +USE_RC_SUBR= ${PORTNAME} + +GO_MODULE= github.com/dsh2dsh/${PORTNAME} +GO_BUILDFLAGS= -ldflags "${STRIP} -w -X github.com/dsh2dsh/${PORTNAME}/version.${PORTNAME}Version=${DISTVERSIONFULL}" + +CONFLICTS_INSTALL= ${PORTNAME} + +SUB_FILES= pkg-message 500.${PORTNAME} + +OPTIONS_DEFINE= EXAMPLES + +post-install: + @${MKDIR} ${STAGEDIR}${EXAMPLESDIR} \ + ${STAGEDIR}${ETCDIR} \ + ${STAGEDIR}${PREFIX}/etc/newsyslog.conf.d \ + ${STAGEDIR}/var/run/${PORTNAME} + ${INSTALL_DATA} ${FILESDIR}/newsyslog.conf \ + ${STAGEDIR}${EXAMPLESDIR}/newsyslog.conf + ${INSTALL_DATA} ${FILESDIR}/${PORTNAME}.yml \ + ${STAGEDIR}${ETCDIR}/${PORTNAME}.yml.sample + ${MKDIR} ${STAGEDIR}${PREFIX}/etc/periodic/weekly + ${INSTALL_SCRIPT} ${WRKDIR}/500.${PORTNAME} \ + ${STAGEDIR}${PREFIX}/etc/periodic/weekly/500.${PORTNAME} + +post-install-EXAMPLES-on: + @${MKDIR} ${STAGEDIR}${EXAMPLESDIR}/hooks + (cd ${WRKSRC}/config/samples && \ + ${COPYTREE_SHARE} . ${STAGEDIR}${EXAMPLESDIR}) + ${INSTALL_DATA} \ + ${WRKSRC}/dist/grafana/grafana-prometheus-${PORTNAME}.json \ + ${STAGEDIR}${EXAMPLESDIR} + +.include <bsd.port.mk> diff --git a/sysutils/zrepl-dsh2dsh/distinfo b/sysutils/zrepl-dsh2dsh/distinfo new file mode 100644 index 000000000000..77e2b6aa4cb4 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/distinfo @@ -0,0 +1,5 @@ +TIMESTAMP = 1718809873 +SHA256 (go/sysutils_zrepl-dsh2dsh/zrepl-v0.7.8/v0.7.8.mod) = 6de73508b106ec921d08bc908dd74e461a0f3b37040791d3945ce48c11a2acbd +SIZE (go/sysutils_zrepl-dsh2dsh/zrepl-v0.7.8/v0.7.8.mod) = 2984 +SHA256 (go/sysutils_zrepl-dsh2dsh/zrepl-v0.7.8/v0.7.8.zip) = 742801772c8511d8d7f03284520437128a70f8536b9722c9e9990335b6f14d17 +SIZE (go/sysutils_zrepl-dsh2dsh/zrepl-v0.7.8/v0.7.8.zip) = 589396 diff --git a/sysutils/zrepl-dsh2dsh/files/500.zrepl.in b/sysutils/zrepl-dsh2dsh/files/500.zrepl.in new file mode 100644 index 000000000000..b7f1b3abb4d3 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/files/500.zrepl.in @@ -0,0 +1,41 @@ +#!/bin/sh + +# Check zrepl SSL certificates for impending expiration each week +# +# Add the following lines to /etc/periodic.conf: +# +# weekly_zrepl_enable (bool): Set to "NO" by default +# weekly_zrepl_warntime (int): Set to one month's worth of seconds by default + +# If there is a global system configuration file, suck it in. +# +if [ -r /etc/defaults/periodic.conf ] +then + . /etc/defaults/periodic.conf + source_periodic_confs +fi + +# 30 days in seconds +: ${weekly_zrepl_warntime="2592000"} + +rc=0 +case "$weekly_zrepl_enable" in + [Yy][Ee][Ss]) + echo + echo "Check Zrepl certificates for upcoming expiration:" + + for cert in `/usr/bin/find %%ETCDIR%% -maxdepth 1 -name *.crt`; do + /usr/bin/openssl x509 --in "${cert}" \ + -checkend "${weekly_zrepl_warntime}" + + if [ $? -gt 0 ]; then + echo "${cert} will expire soon" + /usr/bin/openssl x509 --in "${cert}" -noout -enddate + rc=3 + fi + done + ;; + *) rc=0;; +esac + +exit $rc diff --git a/sysutils/zrepl-dsh2dsh/files/newsyslog.conf b/sysutils/zrepl-dsh2dsh/files/newsyslog.conf new file mode 100644 index 000000000000..82bb8667cdd7 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/files/newsyslog.conf @@ -0,0 +1,5 @@ +# Newsyslog configuration file for zrepl. +# See newsyslog.conf(5) for details. +# +# logfilename [owner:group] mode count size when flags [/pid_file] [sig_num] +/var/log/zrepl.log 644 7 * 24 JNC diff --git a/sysutils/zrepl-dsh2dsh/files/pkg-message.in b/sysutils/zrepl-dsh2dsh/files/pkg-message.in new file mode 100644 index 000000000000..92f9099f1f50 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/files/pkg-message.in @@ -0,0 +1,35 @@ +[ +{ type: install + message: <<EOM +Messages from sysutils/zrepl: + +SETUP BEFORE FIRST USE: +- Create the log file /var/log/zrepl.log: + # service newsyslog restart +- Modify the %%PREFIX%%/etc/zrepl/zrepl.yml configuration file +- Enable the zrepl daemon to start automatically at boot: + # sysrc zrepl_enable="YES" +- Start the zrepl daemon: + # service zrepl start + +Documentation: https://github.com/dsh2dsh/zrepl?tab=readme-ov-file#changes-from-upstream + https://zrepl.github.io/ +Changelog: https://zrepl.github.io/master/changelog.html + +DANGER - SNAPSHOT PRUNING REQUIRES EXPLICIT KEEP RULES: +For any ZFS snapshot that you want to keep, at least one rule must match. +This also applies to snapshots taken by means other than zrepl +(e.g. snapshots taken manually or via boot environment tools). + +In order to automatically warn the operator of impending certificate +expiration, add this line to /etc/periodic.conf: + + weekly_zrepl_enable="YES" + +More config details in the zrepl periodic script: + + %%LOCALBASE%%/etc/periodic/weekly/500.zrepl + +EOM +} +] diff --git a/sysutils/zrepl-dsh2dsh/files/zrepl.in b/sysutils/zrepl-dsh2dsh/files/zrepl.in new file mode 100644 index 000000000000..6a2510f80572 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/files/zrepl.in @@ -0,0 +1,80 @@ +#!/bin/sh + +# PROVIDE: zrepl +# REQUIRE: LOGIN +# KEYWORD: shutdown +# +# Add the following lines to /etc/rc.conf.local or /etc/rc.conf +# to enable this service: +# +# zrepl_enable (bool): Set to YES to enable the zrepl service. +# Default: NO +# zrepl_config (str): File containing zrepl configuration details. +# Default: %%PREFIX%%/etc/zrepl/zrepl.yml +# zrepl_user (str): The user account used to run the zrepl daemon. +# Do not specifically set this to an empty string as this +# will cause the daemon to run as root. +# Default: root +# zrepl_group (str): The group account used to run the zrepl daemon. +# Do not specifically set this to an empty string as this +# will cause the daemon to run with group wheel. +# Default: wheel +# zrepl_flags (str): Extra flags passed to zrepl +# Default: "" +# zrepl_facility (str): Syslog facility to use +# Default: local0 +# zrepl_priority (str): Syslog priority to use +# Default: alert + +. /etc/rc.subr +name=zrepl +rcvar=${name}_enable +load_rc_config $name + +: ${zrepl_enable:="NO"} +: ${zrepl_config:="%%PREFIX%%/etc/zrepl/zrepl.yml"} +: ${zrepl_user:="root"} +: ${zrepl_group:="wheel"} +: ${zrepl_flags:=""} +: ${zrepl_facility:="local0"} +: ${zrepl_priority:="alert"} +: ${zrepl_options:="${zrepl_flags} --config ${zrepl_config}"} + +pidfile="/var/run/zrepl.pid" +command="/usr/sbin/daemon" +procname="%%PREFIX%%/bin/zrepl" +command_args="-p ${pidfile} -S -l ${zrepl_facility} -s ${zrepl_priority} -T ${name} \ + ${procname} ${zrepl_options} daemon" + +start_precmd="zrepl_precmd" +stop_postcmd="zrepl_postcmd" +restart_precmd="zrepl_checkconfig" +configtest_cmd="zrepl_checkconfig" +extra_commands="configtest" + +zrepl_precmd() +{ + if [ ! -d "/var/run/zrepl/stdinserver" ]; then + install -d -g ${zrepl_group} -o ${zrepl_user} -m 0700 -- "/var/run/zrepl"; + install -d -g ${zrepl_group} -o ${zrepl_user} -m 0700 -- "/var/run/zrepl/stdinserver"; + fi + + if [ ! -e "${pidfile}" ]; then + install -g ${zrepl_group} -o ${zrepl_user} -- /dev/null "${pidfile}"; + fi + + zrepl_checkconfig +} + +zrepl_postcmd() +{ + rm -f -- "${pidfile}" +} + +zrepl_checkconfig() +{ + echo "Performing sanity check on zrepl configuration:" + eval ${procname} ${zrepl_options} configcheck +} + +run_rc_command "$1" diff --git a/sysutils/zrepl-dsh2dsh/files/zrepl.yml b/sysutils/zrepl-dsh2dsh/files/zrepl.yml new file mode 100644 index 000000000000..3f9278c345bc --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/files/zrepl.yml @@ -0,0 +1,148 @@ +# zrepl main configuration file. For documentation, refer to +# https://zrepl.github.io/ and https://github.com/dsh2dsh/zrepl +# +global: + # rpc_timeout: "1m" + zfs_bin: "/sbin/zfs" + + logging: + - type: "file" + format: "text" # or "json" + time: false # don't prepend with date and time + hide_fields: &hide-log-fields + - "span" # don't log "span" field + level: "error" # log errors only + # without filename logs to stderr + - type: "file" + format: "text" + hide_fields: *hide-log-fields + level: "info" + filename: "/var/log/zrepl.log" + +# Add "jobs:" section(s) here for your specific use cases. +jobs: + - name: "hourly-snap" + type: "snap" + filesystems: &filesystems + "zroot/ROOT/default<": true + "zroot/usr/home<": true + snapshotting: + type: "periodic" + interval: "1h" + prefix: "zrepl_hourly_" + timestamp_format: "20060102_150405_MST" + timestamp_local: true + pruning: &keep-all + + - name: "daily-snap" + type: "snap" + filesystems: *filesystems + snapshotting: + type: "periodic" + interval: "1d" + prefix: "zrepl_daily_" + timestamp_format: "20060102_150405_MST" + timestamp_local: true + pruning: *keep-all + + - name: "monthly-snap" + type: "snap" + filesystems: *filesystems + snapshotting: + type: "periodic" + interval: "30d" + prefix: "zrepl_monthly_" + timestamp_format: "20060102_150405_MST" + timestamp_local: true + pruning: *keep-all + + - name: "zdisk" + type: "sink" + root_fs: "zdisk/zrepl" + serve: + type: "local" + listener_name: "zdisk" + recv: + placeholder: + encryption: "inherit" + monitor: &monitor-snapshots + latest: + - prefix: "zrepl_frequently_" + critical: "48h" # 2d + - prefix: "zrepl_hourly_" + critical: "48h" + - prefix: "zrepl_daily_" + critical: "48h" + - prefix: "zrepl_monthly_" + critical: "768h" # 32d + oldest: + # see zroot-to-zdisk prunner.keep_receiver + - prefix: "zrepl_frequently_" + critical: "48h" # 2d + - prefix: "zrepl_hourly_" + critical: "168h" # 7d + - prefix: "zrepl_daily_" + critical: "2208h" # 90d + 2d + - prefix: "zrepl_monthly_" + critical: "8688h" # 30 * 12 = 360d + 2d + - prefix: "" # everything else + critical: "168h" # 7d + + - name: "zroot-to-zdisk" + type: "push" + connect: + type: "local" + listener_name: "zdisk" + client_identity: "localhost" + filesystems: *filesystems + # send: + # execpipe: + # # zfs send | mbuffer + # - [ "/usr/local/bin/mbuffer", "-q", "-s", "128k", "-m", "1G" ] + replication: &replication + # Send all intermediary snapshots as a stream package, instead of sending + # them one by one. + #one_step: true + concurrency: + steps: 4 + size_estimates: 8 + conflict_resolution: &conflict_resolution + initial_replication: "all" + snapshotting: + type: "periodic" + cron: "*/15 * * * *" + prefix: "zrepl_frequently_" + timestamp_format: "20060102_150405_MST" + timestamp_local: true + pruning: + keep_sender: + - type: "not_replicated" + - type: "last_n" + count: 16 + regex: "^zrepl_frequently_" + - type: "regex" + negate: true + regex: "^zrepl_" + - type: "last_n" + count: 8 + regex: "^zrepl_hourly_" + - type: "last_n" + count: 4 + regex: "^zrepl_daily_" + - type: "last_n" + count: 2 + regex: "^zrepl_monthly_" + keep_receiver: &keep-receiver + - type: "last_n" + count: 2 + regex: "^zrepl_frequently_" + - type: "last_n" + count: 30 + regex: "^zrepl_hourly_" + - type: "last_n" + count: 90 + regex: "^zrepl_daily_" + - type: "last_n" + count: 12 + regex: "^zrepl_monthly_" + monitor: *monitor-snapshots diff --git a/sysutils/zrepl-dsh2dsh/pkg-descr b/sysutils/zrepl-dsh2dsh/pkg-descr new file mode 100644 index 000000000000..b75b34c591f5 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/pkg-descr @@ -0,0 +1,17 @@ +Cross-platform ZFS replication tool written in Go that supports +feature detection, bookmarks, and other new features. + +This is a fork with more features like + + * support of shell patterns for datasets definitions + * new log formatters + * ability to log into a file + * ability to configure command piplines between `zfs send` and `zfs recv` + * Icinga/Nagios checks + * fast skip "keep all" pruning + * snapshots can be named using local time in timestamps + * configurable RPC timeout + * configurable path to zfs binary + * faster replication + +and so on. diff --git a/sysutils/zrepl-dsh2dsh/pkg-plist b/sysutils/zrepl-dsh2dsh/pkg-plist new file mode 100644 index 000000000000..fc11dec8cfb3 --- /dev/null +++ b/sysutils/zrepl-dsh2dsh/pkg-plist @@ -0,0 +1,23 @@ +bin/zrepl +etc/periodic/weekly/500.zrepl +@sample %%ETCDIR%%/zrepl.yml.sample +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/bandwidth_limit.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/grafana-prometheus-zrepl.json +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/hooks/template.sh +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/local.yml +@sample %%EXAMPLESDIR%%/newsyslog.conf etc/newsyslog.conf.d/zrepl.conf +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/pull.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/pull_ssh.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/push.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/quickstart_backup_to_external_disk.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/quickstart_continuous_server_backup_receiver.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/quickstart_continuous_server_backup_sender.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/quickstart_fan_out_replication_source.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/quickstart_fan_out_replication_target.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/sink.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/snap.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/snap_cron.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/source.yml +%%PORTEXAMPLES%%%%EXAMPLESDIR%%/source_ssh.yml +@dir etc/newsyslog.conf.d +@dir(root,wheel,0750) /var/run/zrepl
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202409162020.48GKKWv6063081>