Date: Mon, 14 Jun 2021 17:46:08 GMT From: Po-Chuan Hsieh <sunpoet@FreeBSD.org> To: ports-committers@FreeBSD.org, dev-commits-ports-all@FreeBSD.org, dev-commits-ports-main@FreeBSD.org Subject: git: 9ff6a0ecc059 - main - dns/py-dns-crawler: Add py-dns-crawler 1.4.9 Message-ID: <202106141746.15EHk8bF052698@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by sunpoet: URL: https://cgit.FreeBSD.org/ports/commit/?id=9ff6a0ecc059e6a27b4a589795a77eb76469183d commit 9ff6a0ecc059e6a27b4a589795a77eb76469183d Author: Po-Chuan Hsieh <sunpoet@FreeBSD.org> AuthorDate: 2021-06-14 17:32:32 +0000 Commit: Po-Chuan Hsieh <sunpoet@FreeBSD.org> CommitDate: 2021-06-14 17:32:32 +0000 dns/py-dns-crawler: Add py-dns-crawler 1.4.9 Despite the name, the crawler gets info for more services than just DNS: - DNS: - all A/AAAA records (for the 2nd level domain and www.subdomain), annotated with GeoIP - TXT records (with SPF and DMARC parsed for easier filtering) - TLSA (for the 2nd level domain and www.subdomain) - MX - DNSSEC validation - nameservers: - each server IP annotated with GeoIP - HOSTNAME.BIND, VERSION.BIND, AUTHORS.BIND and fortune (also for all IPs) - users can add custom additional RRs in the config file - E-mail (for every server from MX): - SMTP server banners (optional, ports are configurable) - TLSA records - Web: - HTTP status & headers (inc. parsed cookies) for ports 80 & 443 on each IP from A/AAAA records - certificate info for HTTPS (optionally with an entire cert chain) - webpage content (optional) - everything of the above is saved for each step in the redirect history -- the crawler follows redirects until it gets a non-redirecting status or hits a configurable limit WWW: https://gitlab.nic.cz/adam/dns-crawler --- dns/Makefile | 1 + dns/py-dns-crawler/Makefile | 39 +++++++++++++++++++++++++++++++ dns/py-dns-crawler/distinfo | 3 +++ dns/py-dns-crawler/files/patch-setup.py | 41 +++++++++++++++++++++++++++++++++ dns/py-dns-crawler/pkg-descr | 25 ++++++++++++++++++++ 5 files changed, 109 insertions(+) diff --git a/dns/Makefile b/dns/Makefile index 10be006cbbee..71c200092077 100644 --- a/dns/Makefile +++ b/dns/Makefile @@ -176,6 +176,7 @@ SUBDIR += py-adns SUBDIR += py-aiodns SUBDIR += py-cloudflare + SUBDIR += py-dns-crawler SUBDIR += py-dns-lexicon SUBDIR += py-dnspython SUBDIR += py-dnspython2 diff --git a/dns/py-dns-crawler/Makefile b/dns/py-dns-crawler/Makefile new file mode 100644 index 000000000000..b0b1ece176ec --- /dev/null +++ b/dns/py-dns-crawler/Makefile @@ -0,0 +1,39 @@ +# Created by: Po-Chuan Hsieh <sunpoet@FreeBSD.org> + +PORTNAME= dns-crawler +PORTVERSION= 1.4.9 +CATEGORIES= dns python +MASTER_SITES= CHEESESHOP +PKGNAMEPREFIX= ${PYTHON_PKGNAMEPREFIX} + +MAINTAINER= sunpoet@FreeBSD.org +COMMENT= Crawler for getting info about DNS domains and services + +LICENSE= GPLv3 +LICENSE_FILE= ${WRKSRC}/LICENSE + +BUILD_DEPENDS= ${PYTHON_PKGNAMEPREFIX}setuptools_scm>=0:devel/py-setuptools_scm@${PY_FLAVOR} +RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}asn1crypto>=1.3.0:devel/py-asn1crypto@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}cert-human>=1.0.7:security/py-cert-human@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}cryptography>=2.8:security/py-cryptography@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}dnspython>=0:dns/py-dnspython@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}ecdsa>=0.15:security/py-ecdsa@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}forcediphttpsadapter>=1.0.1:www/py-forcediphttpsadapter@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}GeoIP2>=3.0.0:net/py-GeoIP2@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}hstspreload>=0:www/py-hstspreload@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}idna>=2.9:dns/py-idna@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}pyaml>=19.12.0:textproc/py-pyaml@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}pycryptodome>=3.9.7:security/py-pycryptodome@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}pyicu>=2.4.3:devel/py-pyicu@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}openssl>=19.1.0:security/py-openssl@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}redis>=3.4.1:databases/py-redis@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}requests>=2.23.0:www/py-requests@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}requests-toolbelt>=0.9.1:www/py-requests-toolbelt@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}rq>=1.2.2:devel/py-rq@${PY_FLAVOR} + +USES= python:3.6+ +USE_PYTHON= autoplist concurrent distutils + +NO_ARCH= yes + +.include <bsd.port.mk> diff --git a/dns/py-dns-crawler/distinfo b/dns/py-dns-crawler/distinfo new file mode 100644 index 000000000000..bd3eda879206 --- /dev/null +++ b/dns/py-dns-crawler/distinfo @@ -0,0 +1,3 @@ +TIMESTAMP = 1622213966 +SHA256 (dns-crawler-1.4.9.tar.gz) = f59340c4ef1769a04a4f9ebc07f27114fae289b91d287433a42c326958afb71d +SIZE (dns-crawler-1.4.9.tar.gz) = 66705 diff --git a/dns/py-dns-crawler/files/patch-setup.py b/dns/py-dns-crawler/files/patch-setup.py new file mode 100644 index 000000000000..dbc26bca9f5c --- /dev/null +++ b/dns/py-dns-crawler/files/patch-setup.py @@ -0,0 +1,41 @@ +--- setup.py.orig 2020-07-10 06:16:32 UTC ++++ setup.py +@@ -39,23 +39,23 @@ setup( + ] + }, + install_requires=[ +- "asn1crypto==1.3.0", +- "cert_human==1.0.7", +- "cryptography==2.8", ++ "asn1crypto>=1.3.0", ++ "cert_human>=1.0.7", ++ "cryptography>=2.8", + "dnspython", +- "ecdsa==0.15", +- "forcediphttpsadapter==1.0.1", +- "geoip2==3.0.0", ++ "ecdsa>=0.15", ++ "forcediphttpsadapter>=1.0.1", ++ "geoip2>=3.0.0", + "hstspreload", +- "idna==2.9", +- "pyaml==19.12.0", +- "PyICU==2.4.3", +- "pycryptodome==3.9.7", +- "pyopenssl==19.1.0", +- "redis==3.4.1", +- "requests_toolbelt==0.9.1", +- "requests==2.23.0", +- "rq==1.2.2", ++ "idna>=2.9", ++ "pyaml>=19.12.0", ++ "PyICU>=2.4.3", ++ "pycryptodome>=3.9.7", ++ "pyopenssl>=19.1.0", ++ "redis>=3.4.1", ++ "requests_toolbelt>=0.9.1", ++ "requests>=2.23.0", ++ "rq>=1.2.2", + ], + keywords=["crawler", "dns", "http", "https"], + classifiers=[ diff --git a/dns/py-dns-crawler/pkg-descr b/dns/py-dns-crawler/pkg-descr new file mode 100644 index 000000000000..b309f887ca79 --- /dev/null +++ b/dns/py-dns-crawler/pkg-descr @@ -0,0 +1,25 @@ +Despite the name, the crawler gets info for more services than just DNS: +- DNS: + - all A/AAAA records (for the 2nd level domain and www.subdomain), annotated + with GeoIP + - TXT records (with SPF and DMARC parsed for easier filtering) + - TLSA (for the 2nd level domain and www.subdomain) + - MX + - DNSSEC validation + - nameservers: + - each server IP annotated with GeoIP + - HOSTNAME.BIND, VERSION.BIND, AUTHORS.BIND and fortune (also for all IPs) + - users can add custom additional RRs in the config file +- E-mail (for every server from MX): + - SMTP server banners (optional, ports are configurable) + - TLSA records +- Web: + - HTTP status & headers (inc. parsed cookies) for ports 80 & 443 on each IP + from A/AAAA records + - certificate info for HTTPS (optionally with an entire cert chain) + - webpage content (optional) + - everything of the above is saved for each step in the redirect history -- + the crawler follows redirects until it gets a non-redirecting status or hits + a configurable limit + +WWW: https://gitlab.nic.cz/adam/dns-crawler
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202106141746.15EHk8bF052698>