From 63ee7cb16ea41848999d917959a67dd4a178d3b4 Mon Sep 17 00:00:00 2001 From: neil Date: Mon, 7 Sep 2026 12:33:21 +0800 Subject: [PATCH 01/17] ci: tolerate 422 when the latest release tag has no workflow_dispatch --- .github/workflows/dockerhub.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 383db8d9..1709d4d6 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -125,6 +125,18 @@ jobs: exit 1 fi echo "dispatching a rebuild of ${tag}" - # fails with 422 when the tag's workflow file has no workflow_dispatch - # trigger (releases before this job existed); nothing to do then - gh workflow run dockerhub.yml --repo "${GITHUB_REPOSITORY}" --ref "${tag}" + # A tag cut before this job existed carries a workflow file with no + # workflow_dispatch trigger; the API rejects the dispatch with 422. + # That is expected (nothing to rebuild there), so only a different + # error fails the job. + if ! out="$(gh workflow run dockerhub.yml --repo "${GITHUB_REPOSITORY}" --ref "${tag}" 2>&1)"; then + echo "$out" + case "$out" in + *"does not have 'workflow_dispatch' trigger"*) + echo "::warning::${tag} predates the dispatch trigger; skipping the rebuild" + ;; + *) + exit 1 + ;; + esac + fi From 162fbde328e009688ffc84b860ba88f9cad3db6c Mon Sep 17 00:00:00 2001 From: jduss Date: Thu, 10 Sep 2026 15:37:03 +0200 Subject: [PATCH 02/17] Update netcup DNS API to support new API (#7214) * dns_netcup: add support for the new netcup REST API Domains managed by the new DNS backend can be handled through the new REST API at api.netcup.com. The API is selected by the length of NC_Apikey: new REST API keys are 64 characters long, legacy CCP API keys are 50. With a REST API key the domain is looked up via GET /v1/domain and the challenge record is managed through the dedicated ACME challenge endpoints. After adding a record, the script waits 20 seconds and then polls until the record reports the deployed status. Domains whose DNS cannot be managed via the REST API yet fall back to the legacy CCP API when NC_Apikey_Legacy, NC_Apipw and NC_CID are configured. * dns_netcup: treat non-challenge records as a no-op on the REST API The REST API can only manage _acme-challenge records, records with other names cannot exist behind it. The DNS-API-Test adds and removes a TXT record outside _acme-challenge and expects both calls to succeed, so treat such records as a successful no-op with an info message instead of failing. * dns_netcup: address review feedback for the REST API support - Only skip the synthetic DNS-API-Test record: real records without the _acme-challenge prefix (e.g. a challenge alias in the "=" form) now fail loudly, or use the legacy CCP API when legacy credentials are configured. The zone walk starts at the full name for them, so an apex alias is found. - Blank _H2..._H5 for REST API calls and clear all header slots before legacy CCP API calls so no auth headers leak between endpoints or dns hooks. - Stop walking the zone lookup when the API reports success:false and surface the response instead of a misleading "no zone found". - Split the response before extracting id/isDnsManaged so the egrep and sed implementations of _egrep_o cannot pick different matches. - Fall back to the legacy CCP API only on a literal isDnsManaged false; error distinctly on an unparsable value. - Poll the deploy status right away and sleep between retries instead of an unconditional 20 second sleep. - Use ${#NC_Apikey} for the key length and rename internal state to _nc_apikey/_nc_endrest. * dns_netcup: walk on when the REST API reports resourceDoesNotExist Querying /domain?fqdn= for a name that is not a domain of the account does not return an empty result: the API answers with success:false and the error code resourceDoesNotExist. Treat exactly that as "not found" during the zone walk and keep failing hard on everything else, e.g. an invalid API key. --- dnsapi/dns_netcup.sh | 400 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 359 insertions(+), 41 deletions(-) diff --git a/dnsapi/dns_netcup.sh b/dnsapi/dns_netcup.sh index 3b291854..87e9b22f 100644 --- a/dnsapi/dns_netcup.sh +++ b/dnsapi/dns_netcup.sh @@ -5,30 +5,315 @@ Domains: netcup.de netcup.net Site: netcup.eu/ Docs: github.com/acmesh-official/acme.sh/wiki/dnsapi#dns_netcup Options: - NC_Apikey API Key - NC_Apipw API Password - NC_CID Customer Number + NC_Apikey API Key. The new netcup REST API key (64 characters) or the legacy CCP API key + NC_Apipw API Password. Only used for the legacy CCP API + NC_CID Customer Number. Only used for the legacy CCP API + NC_Apikey_Legacy Legacy CCP API Key. Only used for domains not manageable via the REST API when NC_Apikey holds a new netcup REST API key. Optional. Author: linux-insideDE ' NC_Apikey="${NC_Apikey:-$(_readaccountconf_mutable NC_Apikey)}" NC_Apipw="${NC_Apipw:-$(_readaccountconf_mutable NC_Apipw)}" NC_CID="${NC_CID:-$(_readaccountconf_mutable NC_CID)}" +NC_Apikey_Legacy="${NC_Apikey_Legacy:-$(_readaccountconf_mutable NC_Apikey_Legacy)}" end="https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON" +_nc_endrest="https://api.netcup.com/v1" client="" dns_netcup_add() { - _debug NC_Apikey "$NC_Apikey" - _login - if [ "$NC_Apikey" = "" ] || [ "$NC_Apipw" = "" ] || [ "$NC_CID" = "" ]; then - _err "No Credentials given" + fulldomain=$1 + txtvalue=$2 + _debug fulldomain "$fulldomain" + _debug txtvalue "$txtvalue" + + if ! _nc_check_credentials; then return 1 fi _saveaccountconf_mutable NC_Apikey "$NC_Apikey" - _saveaccountconf_mutable NC_Apipw "$NC_Apipw" - _saveaccountconf_mutable NC_CID "$NC_CID" + if [ -n "$NC_Apipw" ]; then + _saveaccountconf_mutable NC_Apipw "$NC_Apipw" + fi + if [ -n "$NC_CID" ]; then + _saveaccountconf_mutable NC_CID "$NC_CID" + fi + if [ -n "$NC_Apikey_Legacy" ]; then + _saveaccountconf_mutable NC_Apikey_Legacy "$NC_Apikey_Legacy" + fi + + if _nc_is_rest_key; then + _nc_rest_add "$fulldomain" "$txtvalue" + else + _nc_apikey="$NC_Apikey" + _nc_legacy_add "$fulldomain" "$txtvalue" + fi +} + +dns_netcup_rm() { fulldomain=$1 txtvalue=$2 + _debug fulldomain "$fulldomain" + _debug txtvalue "$txtvalue" + + if ! _nc_check_credentials; then + return 1 + fi + + if _nc_is_rest_key; then + _nc_rest_rm "$fulldomain" "$txtvalue" + else + _nc_apikey="$NC_Apikey" + _nc_legacy_rm "$fulldomain" "$txtvalue" + fi +} + +#################### New netcup REST API (api.netcup.com) #################### + +_nc_rest_add() { + fulldomain=$1 + txtvalue=$2 + + if ! _nc_rest_get_domain "$fulldomain"; then + return 1 + fi + _debug _domain_id "$_domain_id" + _debug _dns_managed "$_dns_managed" + + if [ "$_dns_managed" = "false" ]; then + _nc_rest_use_legacy "$_domain" || return 1 + _nc_legacy_add "$fulldomain" "$txtvalue" + return + fi + if [ "$_dns_managed" != "true" ]; then + _err "Unable to read isDnsManaged for $_domain from the netcup REST API response: $response" + return 1 + fi + + case "$fulldomain" in + _acme-challenge.*) ;; + acmetestXyzRandomName.*) + # The synthetic record of the DNS-API-Test, which expects add and + # rm to succeed. The REST API can only manage _acme-challenge + # records, so skip it. Real records are never treated as a no-op. + _info "Skipping the DNS-API-Test record $fulldomain, the netcup REST API can only manage _acme-challenge records." + return 0 + ;; + *) + # e.g. a challenge alias given in the "=" form without the prefix + if [ -n "$NC_Apikey_Legacy" ] && [ -n "$NC_Apipw" ] && [ -n "$NC_CID" ]; then + _debug "The netcup REST API can only create _acme-challenge records, using the legacy CCP API for $fulldomain" + _nc_apikey="$NC_Apikey_Legacy" + _nc_legacy_add "$fulldomain" "$txtvalue" + return + fi + _err "The netcup REST API can only create _acme-challenge records, unable to create $fulldomain." + _err "Set NC_Apikey_Legacy, NC_Apipw and NC_CID to manage it via the legacy CCP API." + return 1 + ;; + esac + + _nc_rest_get_scope "$fulldomain" "$_domain" + _debug _scope "$_scope" + + if ! _nc_rest POST "domain/$_domain_id/acme/challenge" "{\"scope\": \"$_scope\", \"value\": \"$txtvalue\"}" || + ! _contains "$response" '"success": *true'; then + _err "Unable to add the challenge record: $response" + return 1 + fi + + # The challenge record is added to the zone right away, but deploying + # the zone to the nameservers happens in the background, so poll until + # the record has actually been deployed (up to about 60 seconds). + _nc_tries=0 + while true; do + if _nc_rest GET "domain/$_domain_id/acme/challenge/$_scope/$txtvalue" && + _contains "$response" '"status": *"deployed"'; then + _info "The challenge record has been deployed" + return 0 + fi + _nc_tries=$(_math "$_nc_tries" + 1) + if [ "$_nc_tries" -ge 12 ]; then + break + fi + _debug "The challenge record has not been deployed yet, waiting 5 more seconds" + _sleep 5 + done + _info "The challenge record has still not been deployed after 60 seconds, continuing anyway" + return 0 +} + +_nc_rest_rm() { + fulldomain=$1 + txtvalue=$2 + + if ! _nc_rest_get_domain "$fulldomain"; then + return 1 + fi + + if [ "$_dns_managed" = "false" ]; then + _nc_rest_use_legacy "$_domain" || return 1 + _nc_legacy_rm "$fulldomain" "$txtvalue" + return + fi + if [ "$_dns_managed" != "true" ]; then + _err "Unable to read isDnsManaged for $_domain from the netcup REST API response: $response" + return 1 + fi + + case "$fulldomain" in + _acme-challenge.*) ;; + acmetestXyzRandomName.*) + # See _nc_rest_add. + _info "Skipping the DNS-API-Test record $fulldomain, the netcup REST API can only manage _acme-challenge records." + return 0 + ;; + *) + if [ -n "$NC_Apikey_Legacy" ] && [ -n "$NC_Apipw" ] && [ -n "$NC_CID" ]; then + _debug "The netcup REST API can only remove _acme-challenge records, using the legacy CCP API for $fulldomain" + _nc_apikey="$NC_Apikey_Legacy" + _nc_legacy_rm "$fulldomain" "$txtvalue" + return + fi + _err "The netcup REST API can only remove _acme-challenge records, unable to remove $fulldomain." + _err "Set NC_Apikey_Legacy, NC_Apipw and NC_CID to manage it via the legacy CCP API." + return 1 + ;; + esac + + _nc_rest_get_scope "$fulldomain" "$_domain" + + if ! _nc_rest DELETE "domain/$_domain_id/acme/challenge/$_scope/$txtvalue"; then + _err "Unable to remove the challenge record: $response" + return 1 + fi + _nc_status=$(grep "^HTTP" "$HTTP_HEADER" | _tail_n 1 | cut -d " " -f 2 | tr -d '\r\n') + _debug _nc_status "$_nc_status" + case "$_nc_status" in + 204) + return 0 + ;; + 404) + _info "The challenge record was not found, nothing to remove" + return 0 + ;; + *) + _err "Unable to remove the challenge record: $response" + return 1 + ;; + esac +} + +# fulldomain +# Sets _domain_id, _domain and _dns_managed of the domain the record +# belongs to, walking up the name, longest match first. For a challenge +# record the leftmost label is the prefix and can never be a zone, so +# the walk starts one label in. Other names (e.g. a challenge alias in +# the "=" form) may be a zone apex themselves. +_nc_rest_get_domain() { + case "$1" in + _acme-challenge.*) i=2 ;; + *) i=1 ;; + esac + while true; do + h=$(printf "%s" "$1" | cut -d . -f "$i"-100) + if [ -z "$h" ]; then + _nc_nozone "$1" + return 1 + fi + _debug h "$h" + if ! _nc_rest GET "domain?fqdn=$h"; then + return 1 + fi + if _contains "$response" '"success": *true'; then + if _contains "$response" '"fqdn"'; then + # split the response so that first/last match cannot differ + # between the egrep and sed implementations of _egrep_o + _domain_id=$(printf "%s" "$response" | tr '{,' '\n' | _egrep_o '"id": *[0-9][0-9]*' | _head_n 1 | tr -dc '0-9') + _dns_managed=$(printf "%s" "$response" | tr '{,' '\n' | _egrep_o '"isDnsManaged": *[a-z][a-z]*' | _head_n 1 | sed 's/.*: *//') + _domain="$h" + if [ -n "$_domain_id" ]; then + return 0 + fi + _err "Unable to parse the domain id from the netcup REST API response: $response" + return 1 + fi + # an empty result, $h is not a domain of this account: walk on + elif _contains "$response" '"code": *"resourceDoesNotExist"'; then + # the API reports a domain that is not in this account with + # success:false and this error code: walk on + _debug "$h is not a domain of this account" + else + # e.g. an invalid API key; do not walk on, it would end in a + # misleading "no zone found" error + _err "The netcup REST API request failed: $response" + _err "Note: NC_Apikey was detected as a netcup REST API key because it is 64 characters long." + return 1 + fi + i=$(_math "$i" + 1) + done +} + +# fulldomain domain +# Sets _scope to the host part of the challenge relative to the domain. +# The REST API prepends _acme-challenge. to the scope itself, so the +# prefix is stripped from the record name (the callers guarantee it is +# present). +_nc_rest_get_scope() { + _scope="${1#_acme-challenge.}" + if [ "$_scope" = "$2" ]; then + _scope="@" + else + _scope="${_scope%".$2"}" + fi +} + +# domain +# Selects the legacy credentials for a domain whose DNS cannot be +# managed via the new netcup REST API. +_nc_rest_use_legacy() { + _debug "The DNS of $1 cannot be managed via the REST API, using the legacy CCP API" + if [ -z "$NC_Apikey_Legacy" ] || [ -z "$NC_Apipw" ] || [ -z "$NC_CID" ]; then + _err "The DNS of the domain $1 cannot be managed via the new netcup REST API." + _err "Set NC_Apikey_Legacy, NC_Apipw and NC_CID to your legacy CCP API credentials to manage it." + return 1 + fi + _nc_apikey="$NC_Apikey_Legacy" +} + +# method endpoint [data] +# The response is returned in the global variable $response. +_nc_rest() { + m=$1 + ep=$2 + data=$3 + _debug2 "REST $m $ep" + + export _H1="Authorization: Bearer $NC_Apikey" + # blank the remaining header slots so that auth headers of another + # dns hook cannot ride into the netcup REST API in a multi-provider + # issuance + export _H2="" + export _H3="" + export _H4="" + export _H5="" + if [ "$m" = "GET" ]; then + response=$(_get "$_nc_endrest/$ep") + else + _debug2 data "$data" + response=$(_post "$data" "$_nc_endrest/$ep" "" "$m" "application/json") + fi + _nc_ret="$?" + _debug2 response "$response" + return "$_nc_ret" +} + +#################### Legacy CCP API (ccp.netcup.net) #################### + +_nc_legacy_add() { + fulldomain=$1 + txtvalue=$2 + if ! _nc_legacy_login; then + return 1 + fi domain="" exit=$(echo "$fulldomain" | tr -dc '.' | wc -c) exit=$(_math "$exit" + 1) @@ -46,7 +331,7 @@ dns_netcup_add() { domain="$tmp.$domain" fi if [ "$(_math "$i" - "$exit")" -ge 1 ]; then - msg=$(_post "{\"action\": \"updateDnsRecords\", \"param\": {\"apikey\": \"$NC_Apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\",\"clientrequestid\": \"$client\" , \"domainname\": \"$domain\", \"dnsrecordset\": { \"dnsrecords\": [ {\"id\": \"\", \"hostname\": \"$fulldomain.\", \"type\": \"TXT\", \"priority\": \"\", \"destination\": \"$txtvalue\", \"deleterecord\": \"false\", \"state\": \"yes\"} ]}}}" "$end" "" "POST") + msg=$(_post "{\"action\": \"updateDnsRecords\", \"param\": {\"apikey\": \"$_nc_apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\",\"clientrequestid\": \"$client\" , \"domainname\": \"$domain\", \"dnsrecordset\": { \"dnsrecords\": [ {\"id\": \"\", \"hostname\": \"$fulldomain.\", \"type\": \"TXT\", \"priority\": \"\", \"destination\": \"$txtvalue\", \"deleterecord\": \"false\", \"state\": \"yes\"} ]}}}" "$end" "" "POST") _debug "$msg" if [ "$(_getfield "$msg" "5" | sed 's/"statuscode"://g')" != 5028 ]; then if [ "$(_getfield "$msg" "4" | sed s/\"status\":\"//g | sed s/\"//g)" != "success" ]; then @@ -65,13 +350,15 @@ dns_netcup_add() { _nc_nozone "$fulldomain" return 1 fi - logout + _nc_legacy_logout } -dns_netcup_rm() { - _login +_nc_legacy_rm() { fulldomain=$1 txtvalue=$2 + if ! _nc_legacy_login; then + return 1 + fi domain="" exit=$(echo "$fulldomain" | tr -dc '.' | wc -c) @@ -91,7 +378,7 @@ dns_netcup_rm() { domain="$tmp.$domain" fi if [ "$(_math "$i" - "$exit")" -ge 1 ]; then - msg=$(_post "{\"action\": \"infoDnsRecords\", \"param\": {\"apikey\": \"$NC_Apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\", \"domainname\": \"$domain\"}}" "$end" "" "POST") + msg=$(_post "{\"action\": \"infoDnsRecords\", \"param\": {\"apikey\": \"$_nc_apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\", \"domainname\": \"$domain\"}}" "$end" "" "POST") rec=$(echo "$msg" | sed 's/\[//g' | sed 's/\]//g' | sed 's/{\"serverrequestid\".*\"dnsrecords\"://g' | sed 's/},{/};{/g' | sed 's/{//g' | sed 's/}//g') _debug "$msg" if [ "$(_getfield "$msg" "5" | sed 's/"statuscode"://g')" != 5028 ]; then @@ -132,21 +419,70 @@ dns_netcup_rm() { i=0 fi done - msg=$(_post "{\"action\": \"updateDnsRecords\", \"param\": {\"apikey\": \"$NC_Apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\",\"clientrequestid\": \"$client\" , \"domainname\": \"$domain\", \"dnsrecordset\": { \"dnsrecords\": [ {\"id\": \"$ids\", \"hostname\": \"$fulldomain.\", \"type\": \"TXT\", \"priority\": \"\", \"destination\": \"$txtvalue\", \"deleterecord\": \"TRUE\", \"state\": \"yes\"} ]}}}" "$end" "" "POST") + msg=$(_post "{\"action\": \"updateDnsRecords\", \"param\": {\"apikey\": \"$_nc_apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\",\"clientrequestid\": \"$client\" , \"domainname\": \"$domain\", \"dnsrecordset\": { \"dnsrecords\": [ {\"id\": \"$ids\", \"hostname\": \"$fulldomain.\", \"type\": \"TXT\", \"priority\": \"\", \"destination\": \"$txtvalue\", \"deleterecord\": \"TRUE\", \"state\": \"yes\"} ]}}}" "$end" "" "POST") _debug "$msg" if [ "$(_getfield "$msg" "4" | sed s/\"status\":\"//g | sed s/\"//g)" != "success" ]; then _err "$msg" return 1 fi - logout + _nc_legacy_logout } -# The zone is looked up by walking the challenge name from the right, one -# label at a time. The leftmost label is the challenge prefix, so the full -# name itself can never be a zone: asking netcup for it only returns 4013 -# "Validation Error", which would then mask the real 5028 "zone could not be -# found". Stop one label short, unless the name is too short to have a -# challenge prefix at all (manual invocation). +_nc_legacy_login() { + # never send the REST API Bearer header (or auth headers of another + # dns hook) to the legacy CCP API + export _H1="" + export _H2="" + export _H3="" + export _H4="" + export _H5="" + tmp=$(_post "{\"action\": \"login\", \"param\": {\"apikey\": \"$_nc_apikey\", \"apipassword\": \"$NC_Apipw\", \"customernumber\": \"$NC_CID\"}}" "$end" "" "POST") + sid=$(echo "$tmp" | tr '{}' '\n' | grep apisessionid | cut -d '"' -f 4) + _debug "$tmp" + if [ "$(_getfield "$tmp" "4" | sed s/\"status\":\"//g | sed s/\"//g)" != "success" ]; then + _err "$tmp" + return 1 + fi +} + +_nc_legacy_logout() { + tmp=$(_post "{\"action\": \"logout\", \"param\": {\"apikey\": \"$_nc_apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\"}}" "$end" "" "POST") + _debug "$tmp" + if [ "$(_getfield "$tmp" "4" | sed s/\"status\":\"//g | sed s/\"//g)" != "success" ]; then + _err "$tmp" + return 1 + fi +} + +#################### Shared helpers #################### + +_nc_check_credentials() { + if [ -z "$NC_Apikey" ]; then + _err "No Credentials given" + _err "Set NC_Apikey to your netcup REST API key (64 characters) or your legacy CCP API key." + return 1 + fi + if ! _nc_is_rest_key; then + if [ -z "$NC_Apipw" ] || [ -z "$NC_CID" ]; then + _err "No Credentials given" + _err "The legacy CCP API needs NC_Apikey, NC_Apipw and NC_CID." + return 1 + fi + fi +} + +# New netcup REST API keys are 64 characters long, legacy CCP API keys +# are 50, so the key length selects the API. +_nc_is_rest_key() { + [ "${#NC_Apikey}" -eq 64 ] +} + +# The legacy zone lookup walks the challenge name from the right, one +# label at a time. The leftmost label is the challenge prefix, so the +# full name itself can never be a zone: asking netcup for it only +# returns 4013 "Validation Error", which would then mask the real 5028 +# "zone could not be found". Stop one label short, unless the name is +# too short to have a challenge prefix at all (manual invocation). # levels _nc_lastlevel() { if [ "$1" -ge 3 ]; then @@ -159,23 +495,5 @@ _nc_lastlevel() { # fulldomain _nc_nozone() { _err "No DNS zone for $1 was found at netcup." - _err "Check that the domain belongs to the account of the configured NC_CID and that its DNS is hosted at netcup." -} - -_login() { - tmp=$(_post "{\"action\": \"login\", \"param\": {\"apikey\": \"$NC_Apikey\", \"apipassword\": \"$NC_Apipw\", \"customernumber\": \"$NC_CID\"}}" "$end" "" "POST") - sid=$(echo "$tmp" | tr '{}' '\n' | grep apisessionid | cut -d '"' -f 4) - _debug "$tmp" - if [ "$(_getfield "$tmp" "4" | sed s/\"status\":\"//g | sed s/\"//g)" != "success" ]; then - _err "$tmp" - return 1 - fi -} -logout() { - tmp=$(_post "{\"action\": \"logout\", \"param\": {\"apikey\": \"$NC_Apikey\", \"apisessionid\": \"$sid\", \"customernumber\": \"$NC_CID\"}}" "$end" "" "POST") - _debug "$tmp" - if [ "$(_getfield "$tmp" "4" | sed s/\"status\":\"//g | sed s/\"//g)" != "success" ]; then - _err "$tmp" - return 1 - fi + _err "Check that the domain belongs to the account of the configured credentials and that its DNS is hosted at netcup." } From 0fa785be2a644156a49b7d10190225ca0a398eb1 Mon Sep 17 00:00:00 2001 From: Tok-Ra85 Date: Thu, 10 Sep 2026 08:39:13 -0500 Subject: [PATCH 03/17] TrueNAS deploy script witch websocat instead of Python midclt internal command (#7216) * Add files via upload TrueNAS deploy script for SCALE/CORE using websocket (websocat binary) It is recommend to use a wildcard certificate Tested with TrueNAS SCALE 25.10 (API "wss://host/api/current", JSON-RPC 2.0). Unlike "truenas_ws" hook, this script does NOT use midclt, the truenas_api_client Python package. It only depends on: - jq - websocat (a static binary you deploy) Why: avoids installing a Python environment / TrueNAS package on OPNsense just to push a certificate. IMPORTANT: This script is written in pure POSIX sh (no coproc, no bash arrays). * Update truenas_websocat.sh Mistake on port and procotol. * Update truenas_websocat.sh Adjustment on the "Why" * Add files via upload * Update truenas_websocat.sh * Update truenas_websocat.sh Apply shellcheck disable=SC2016 to avoid false positive. * Update truenas_websocat.sh --- deploy/truenas_websocat.sh | 518 +++++++++++++++++++++++++++++++++++++ 1 file changed, 518 insertions(+) create mode 100644 deploy/truenas_websocat.sh diff --git a/deploy/truenas_websocat.sh b/deploy/truenas_websocat.sh new file mode 100644 index 00000000..c5c7db2e --- /dev/null +++ b/deploy/truenas_websocat.sh @@ -0,0 +1,518 @@ +#!/usr/bin/env sh +# shellcheck disable=SC2016 +# TrueNAS deploy script for SCALE/CORE using websocket (websocat binary) +# It is recommend to use a wildcard certificate +# +# Tested with TrueNAS SCALE 25.10 (API "wss://host/api/current", JSON-RPC 2.0). +# +# Unlike "truenas_ws" hook, this script does NOT use midclt, the truenas_api_client Python package. +# It only depends on: +# - jq +# - websocat (a static binary you deploy) +# +# Why: avoids installing a Python environment / TrueNAS package on remote machine just to push a certificate. +# +# IMPORTANT: This script is written in pure POSIX sh (no coproc, no bash arrays). +# +# +# --------------------------------------------------------------------------- +# Environment variables +# --------------------------------------------------------------------------- +# +# # Use the folowing URL to create a new API token: /ui/apikeys +# +# Required: +# export DEPLOY_TRUENAS_APIKEY="" +# +# Optional: +# export DEPLOY_TRUENAS_HOSTNAME="" (required on first run) +# export DEPLOY_TRUENAS_PROTOCOL="ws" # ws or wss (default: ws) +# export DEPLOY_TRUENAS_PORT="80" # 80, 443, 8443 (default: 80) +# NOTE: defaults are intentionally "ws"/80, not "wss"/443: a freshly +# installed TrueNAS serves its Web UI over plain HTTP on port 80 out +# of the box, and port 443 is not listening until HTTPS is configured. +# Port 80 stays reachable even after HTTPS is enabled, so this keeps +# the hook working on first run without extra setup. +# export DEPLOY_TRUENAS_UPDATE_FTP="no" # yes or no (default: no) also updates the FTP certificate +# export DEPLOY_TRUENAS_UPDATE_APPS="no" # yes or no (default: no) also updates the certificate for any +# iX app exposing a "certificate_id" option. +# WARNING: this redeploys (restarts) every matching app. +# --------------------------------------------------------------------------- + +######################## +### Public functions ### +######################## + +# truenas_websocat_deploy +# +# Deploy new certificate to TrueNAS services with websocat binary +# +# Arguments +# 1: Domain +# 2: Key-File +# 3: Certificate-File +# 4: CA-File +# 5: FullChain-File +# Returns: +# 0: Success +# 1: Missing or invalid API Key +# 2: TrueNAS not ready (health check failed) +# 3: (reserved) +# 4: FTP & iX App cert error +# 5: WebUI cert error +# 6: Certificate creation job error +# 7: Websocat / transport call error (socket write/read failed) +# 8: Missing binary or invalid configuration +# 9: TrueNAS API returned an explicit error (JSON-RPC .error field) + +truenas_websocat_deploy() { + _jq_bin=$(command -v jq 2>/dev/null) + if [ -z "$_jq_bin" ]; then + _err "jq binary not found in PATH. Install it using your system's package manager." + return 8 + fi + _websocat_bin=$(command -v websocat 2>/dev/null) + if [ -z "$_websocat_bin" ]; then + _err "websocat binary not found in PATH. Install it using your system's package manager, or download a static binary from https://github.com/vi/websocat/releases." + return 8 + fi + + _domain="$1" + _file_key="$2" + _file_cert="$3" + _file_cca="$4" + _file_fullchain="$5" + _debug _domain "$_domain" + _debug _file_key "$_file_key" + _debug _file_cert "$_file_cert" + _debug _file_ca "$_file_cca" + _debug _file_fullchain "$_file_fullchain" + + if [ ! -x "$_jq_bin" ]; then + _err "Binary not found or not executable: $_jq_bin" + return 8 + fi + if [ ! -x "$_websocat_bin" ]; then + _err "Binary not found or not executable: $_websocat_bin" + return 8 + fi + + ### ---- Configuration ---- + + _info "Checking environment variables..." + _getdeployconf DEPLOY_TRUENAS_APIKEY + _getdeployconf DEPLOY_TRUENAS_HOSTNAME + _getdeployconf DEPLOY_TRUENAS_PROTOCOL + _getdeployconf DEPLOY_TRUENAS_PORT + _getdeployconf DEPLOY_TRUENAS_UPDATE_FTP + _getdeployconf DEPLOY_TRUENAS_UPDATE_APPS + + # Check API Key + if [ -z "$DEPLOY_TRUENAS_APIKEY" ]; then + _err "TrueNAS API key not found, please set the DEPLOY_TRUENAS_APIKEY environment variable." + return 1 + fi + # Check Hostname, default to localhost if not set + if [ -z "$DEPLOY_TRUENAS_HOSTNAME" ]; then + _info "TrueNAS hostname not set. Using 'localhost'." + DEPLOY_TRUENAS_HOSTNAME="localhost" + fi + + # Check protocol, default to ws if not set: a freshly installed TrueNAS serves its Web UI over plain HTTP, so wss/443 is not available out of the box. + # Use DEPLOY_TRUENAS_PROTOCOL="wss" once HTTPS is configured, since the payload otherwise carries the API key and private key in plain text. + if [ -z "$DEPLOY_TRUENAS_PROTOCOL" ]; then + _info "TrueNAS protocol not set. Using 'ws'." + DEPLOY_TRUENAS_PROTOCOL="ws" + fi + + # Check port, default to 80 if not set (see protocol comment above) + if [ -z "$DEPLOY_TRUENAS_PORT" ]; then + _info "TrueNAS port not set. Using '80'." + DEPLOY_TRUENAS_PORT="80" + fi + case "$DEPLOY_TRUENAS_PORT" in + '' | *[!0-9]*) + _err "Invalid TrueNAS port '$DEPLOY_TRUENAS_PORT'. DEPLOY_TRUENAS_PORT must be numeric." + return 8 + ;; + esac + + _truenas_websocat_uri="$DEPLOY_TRUENAS_PROTOCOL://$DEPLOY_TRUENAS_HOSTNAME:$DEPLOY_TRUENAS_PORT/api/current" + + # Check FTP update, default to no if not set + if [ -z "$DEPLOY_TRUENAS_UPDATE_FTP" ]; then + _info "Certificate update for FTP is not set. Using 'no'." + DEPLOY_TRUENAS_UPDATE_FTP="no" + fi + + # Check Apps update, default to no if not set + if [ -z "$DEPLOY_TRUENAS_UPDATE_APPS" ]; then + _info "Certificate update for Apps is not set. Using 'no'." + DEPLOY_TRUENAS_UPDATE_APPS="no" + fi + + _debug2 DEPLOY_TRUENAS_HOSTNAME "$DEPLOY_TRUENAS_HOSTNAME" + _debug2 DEPLOY_TRUENAS_PROTOCOL "$DEPLOY_TRUENAS_PROTOCOL" + _debug2 DEPLOY_TRUENAS_UPDATE_FTP "$DEPLOY_TRUENAS_UPDATE_FTP" + _debug2 DEPLOY_TRUENAS_UPDATE_APPS "$DEPLOY_TRUENAS_UPDATE_APPS" + _debug _truenas_websocat_uri "$_truenas_websocat_uri" + _secure_debug2 DEPLOY_TRUENAS_APIKEY "$DEPLOY_TRUENAS_APIKEY" + _info "Environment variables: OK" + + ### ---- Persistent WebSocket connection (FIFOs, sh/dash compatible) ---- + # + # Authentication is tied to the WebSocket connection: + # the SAME connection must stay open from login until the end, otherwise every subsequent call comes back unauthenticated. + # We use two FIFOs + `exec` to talk to a background websocat process, without relying on bash-only extensions. + + _websocat_tmpdir=$(mktemp -d /tmp/truenas_websocat.XXXXXX) || { + _err "mktemp failed" + return 3 + } + _websocat_fifo_in="${_websocat_tmpdir}/in" + _websocat_fifo_out="${_websocat_tmpdir}/out" + mkfifo "$_websocat_fifo_in" "$_websocat_fifo_out" || { + _err "mkfifo failed" + rm -rf "$_websocat_tmpdir" + return 3 + } + + "$_websocat_bin" -n -k "$_truenas_websocat_uri" <"$_websocat_fifo_in" >"$_websocat_fifo_out" 2>"${_websocat_tmpdir}/err.log" & + _websocat_pid=$! + + # Opening "in" for read+write avoids a deadlock if websocat hasn't opened the fifo for reading yet at the time we write to it. + exec 3<>"$_websocat_fifo_in" + exec 4<"$_websocat_fifo_out" + + sleep 1 + if ! kill -0 "$_websocat_pid" 2>/dev/null; then + _err "websocat exited prematurely." + _err "$(cat "${_websocat_tmpdir}/err.log" 2>/dev/null)" + exec 3>&- 4<&- + rm -rf "$_websocat_tmpdir" + return 3 + fi + + _websocat_req_counter=0 + + _truenas_websocat_cleanup() { + exec 3>&- 2>/dev/null + exec 4<&- 2>/dev/null + [ -n "$_websocat_pid" ] && kill "$_websocat_pid" 2>/dev/null + rm -rf "$_websocat_tmpdir" 2>/dev/null + } + + # _truenas_websocat_rpc_call + # Does NOT log the payload/response: some calls (certificate.create, core.get_jobs) contain the certificate and private key in plain text, which would massively bloat the logs. + _truenas_websocat_rpc_call() { + _truenas_websocat_method="$1" + _truenas_websocat_params="$2" + _websocat_req_counter=$((_websocat_req_counter + 1)) + _req_id="$_websocat_req_counter" + + _truenas_websocat_payload=$("$_jq_bin" -c -n \ + --arg jsonrpc "2.0" \ + --arg id "$_req_id" \ + --arg method "$_truenas_websocat_method" \ + --argjson params "$_truenas_websocat_params" \ + '{jsonrpc: $jsonrpc, id: $id, method: $method, params: $params}') + + printf '%s\n' "$_truenas_websocat_payload" >&3 || { + _err "Socket write failed (method: $_truenas_websocat_method)" + return 7 + } + IFS= read -r _truenas_websocat_response <&4 || { + _err "Socket read failed (method: $_truenas_websocat_method)" + return 7 + } + + printf '%s' "$_truenas_websocat_response" + } + + # _truenas_websocat_rpc_has_error