Skip to content

Commit d042aaf

Browse files
committed
Add retry and timeout for stuck brew calls
1 parent f97ca00 commit d042aaf

6 files changed

Lines changed: 125 additions & 12 deletions

File tree

src/scripts/darwin.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ handle_dependency_extensions() {
1515
brew_opts=(-sf)
1616
patch_abstract_file >/dev/null 2>&1
1717
for dependency_extension in "${dependency_extensions[@]}"; do
18-
brew install "${brew_opts[@]}" "$ext_tap/$dependency_extension@$version" >/dev/null 2>&1 && copy_brew_extensions "$dependency_extension"
18+
safe_brew install "${brew_opts[@]}" "$ext_tap/$dependency_extension@$version" >/dev/null 2>&1 && copy_brew_extensions "$dependency_extension"
1919
done
2020
fi
2121
}
@@ -83,7 +83,7 @@ add_brew_extension() {
8383
formula="$(get_renamed_formula "$formula")"
8484
update_dependencies >/dev/null 2>&1
8585
handle_dependency_extensions "$formula" "$extension" >/dev/null 2>&1
86-
(brew install "${brew_opts[@]}" "$ext_tap/$formula@$version" >/dev/null 2>&1 && copy_brew_extensions "$formula") || pecl_install "$extension" >/dev/null 2>&1
86+
(safe_brew install "${brew_opts[@]}" "$ext_tap/$formula@$version" >/dev/null 2>&1 && copy_brew_extensions "$formula") || pecl_install "$extension" >/dev/null 2>&1
8787
add_extension_log "$extension" "Installed and enabled"
8888
fi
8989
}
@@ -181,14 +181,14 @@ add_php() {
181181
fi
182182
if [[ "$existing_version" != "false" && -z "$suffix" ]]; then
183183
if [ "$action" = "upgrade" ]; then
184-
brew install --only-dependencies "$php_formula"
185-
brew upgrade -f --overwrite "$php_formula"
184+
safe_brew install --only-dependencies "$php_formula"
185+
safe_brew upgrade -f --overwrite "$php_formula"
186186
else
187187
brew unlink "$php_keg"
188188
fi
189189
else
190-
brew install --only-dependencies "$php_formula"
191-
brew install -f --overwrite "$php_formula" 2>/dev/null || brew upgrade -f --overwrite "$php_formula"
190+
safe_brew install --only-dependencies "$php_formula"
191+
safe_brew install -f --overwrite "$php_formula" 2>/dev/null || safe_brew upgrade -f --overwrite "$php_formula"
192192
fi
193193
brew link --force --overwrite "$php_keg" || (sudo chown -R "$(id -un)":"$(id -gn)" "$brew_prefix" && brew link --force --overwrite "$php_keg")
194194
}

src/scripts/extensions/relay.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ get_openssl_suffix() {
3636
change_library_paths() {
3737
if [ "$os" = "Darwin" ]; then
3838
otool -L "${ext_dir:?}"/relay.so | grep -q 'ssl.1' && openssl_version='1.1' || openssl_version='3'
39-
[ -e "${brew_prefix:?}"/opt/openssl@"$openssl_version" ] || brew install openssl@"$openssl_version"
39+
[ -e "${brew_prefix:?}"/opt/openssl@"$openssl_version" ] || safe_brew install openssl@"$openssl_version"
4040
dylibs="$(otool -L "${ext_dir:?}"/relay.so | grep -Eo '.*\.dylib' | cut -f1 -d ' ')"
4141
install_name_tool -change "$(echo "${dylibs}" | grep -E "libzstd.*dylib" | xargs)" "$brew_prefix"/opt/zstd/lib/libzstd.dylib "$ext_dir"/relay.so
4242
install_name_tool -change "$(echo "${dylibs}" | grep -E "liblz4.*dylib" | xargs)" "$brew_prefix"/opt/lz4/lib/liblz4.dylib "$ext_dir"/relay.so
@@ -54,7 +54,7 @@ add_relay_dependencies() {
5454
if [ "$os" = "Darwin" ]; then
5555
. "${0%/*}"/tools/brew.sh
5656
configure_brew
57-
brew install lz4 hiredis zstd concurrencykit
57+
safe_brew install lz4 hiredis zstd concurrencykit
5858
fi
5959
}
6060

src/scripts/extensions/source.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ add_linux_libs() {
6060
add_darwin_libs() {
6161
local lib=$1
6262
if ! check_lib "$lib"; then
63-
brew install "$lib" >/dev/null 2>&1 || true
63+
safe_brew install "$lib" >/dev/null 2>&1 || true
6464
if [[ "$lib" = *@* ]]; then
6565
brew link --overwrite --force "$lib" >/dev/null 2>&1 || true
6666
fi

src/scripts/tools/blackfire.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ add_blackfire_linux() {
88
add_blackfire_darwin() {
99
sudo mkdir -p /usr/local/var/run
1010
add_brew_tap blackfireio/homebrew-blackfire
11-
brew install blackfire
11+
safe_brew install blackfire
1212
}
1313

1414
blackfire_config() {

src/scripts/tools/brew.sh

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,118 @@ add_brew_bins_to_path() {
4444
add_path "$brew_prefix"/sbin
4545
}
4646

47+
# Function to get file modification time.
48+
get_file_mtime() {
49+
local file=$1
50+
if [ "$(uname -s)" = "Darwin" ]; then
51+
stat -f "%m" "$file" 2>/dev/null || echo 0
52+
else
53+
stat -c "%Y" "$file" 2>/dev/null || echo 0
54+
fi
55+
}
56+
57+
# Function to terminate a process and its direct children.
58+
terminate_process_tree() {
59+
local pid=$1
60+
local children child
61+
children=$(pgrep -P "$pid" 2>/dev/null || true)
62+
kill -TERM "$pid" >/dev/null 2>&1 || true
63+
for child in $children; do
64+
terminate_process_tree "$child"
65+
done
66+
sleep 2
67+
kill -KILL "$pid" >/dev/null 2>&1 || true
68+
for child in $children; do
69+
terminate_process_tree "$child"
70+
done
71+
}
72+
73+
# Function to run a command with an inactivity watchdog.
74+
run_with_inactivity_watchdog() {
75+
local timeout_secs="${SETUP_PHP_BREW_INACTIVITY_TIMEOUT:-180}"
76+
local poll_secs="${SETUP_PHP_BREW_WATCHDOG_POLL:-5}"
77+
local tmp_dir fifo log_file timeout_file command_pid reader_pid monitor_pid exit_code
78+
tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/setup-php-brew.XXXXXX")" || return 1
79+
fifo="$tmp_dir/output.fifo"
80+
log_file="$tmp_dir/output.log"
81+
timeout_file="$tmp_dir/timed_out"
82+
mkfifo "$fifo" || {
83+
rm -rf "$tmp_dir"
84+
return 1
85+
}
86+
: >"$log_file"
87+
88+
("$@" >"$fifo" 2>&1) &
89+
command_pid=$!
90+
91+
(
92+
while IFS= read -r line || [ -n "$line" ]; do
93+
printf '%s\n' "$line"
94+
printf '%s\n' "$line" >>"$log_file"
95+
done <"$fifo"
96+
) &
97+
reader_pid=$!
98+
99+
(
100+
local last_activity current_activity now
101+
last_activity=$(get_file_mtime "$log_file")
102+
while kill -0 "$command_pid" >/dev/null 2>&1; do
103+
sleep "$poll_secs"
104+
current_activity=$(get_file_mtime "$log_file")
105+
[ "$current_activity" -gt "$last_activity" ] && last_activity="$current_activity"
106+
now=$(date +%s)
107+
if [ $((now - last_activity)) -ge "$timeout_secs" ]; then
108+
printf "\nsetup-php: brew produced no output for %ss; terminating and retrying...\n" "$timeout_secs" >&2
109+
: >"$timeout_file"
110+
terminate_process_tree "$command_pid"
111+
break
112+
fi
113+
done
114+
) &
115+
monitor_pid=$!
116+
117+
wait "$command_pid"
118+
exit_code=$?
119+
wait "$reader_pid" 2>/dev/null || true
120+
kill "$monitor_pid" >/dev/null 2>&1 || true
121+
wait "$monitor_pid" 2>/dev/null || true
122+
123+
if [ -e "$timeout_file" ]; then
124+
rm -rf "$tmp_dir"
125+
return 124
126+
fi
127+
128+
rm -rf "$tmp_dir"
129+
return "$exit_code"
130+
}
131+
132+
# Function to run brew with retries and an inactivity watchdog.
133+
safe_brew() {
134+
local max_attempts="${SETUP_PHP_BREW_RETRY_ATTEMPTS:-3}"
135+
local attempt=1
136+
local exit_code=0
137+
138+
if [ "${SETUP_PHP_BREW_WATCHDOG:-true}" = "false" ]; then
139+
brew "$@"
140+
return $?
141+
fi
142+
143+
while [ "$attempt" -le "$max_attempts" ]; do
144+
run_with_inactivity_watchdog brew "$@" && return 0
145+
exit_code=$?
146+
147+
if [ "$attempt" -ge "$max_attempts" ]; then
148+
return "$exit_code"
149+
fi
150+
151+
printf "setup-php: retrying brew command (attempt %s/%s, exit %s)\n" "$((attempt + 1))" "$max_attempts" "$exit_code" >&2
152+
sleep "$((attempt * 5))"
153+
attempt=$((attempt + 1))
154+
done
155+
156+
return "$exit_code"
157+
}
158+
47159
# Function to add brew.
48160
add_brew() {
49161
brew_prefix="$(get_brew_prefix)"
@@ -74,6 +186,7 @@ configure_brew() {
74186
export HOMEBREW_NO_ENV_HINTS=1
75187
export HOMEBREW_NO_INSTALL_CLEANUP=1
76188
export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
189+
export HOMEBREW_DOWNLOAD_CONCURRENCY="${HOMEBREW_DOWNLOAD_CONCURRENCY:-6}"
77190
export brew_opts
78191
export brew_path
79192
export brew_path_dir

src/scripts/tools/grpc_php_plugin.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ add_bazel() {
44
add_list bazel/apt https://storage.googleapis.com/bazel-apt https://bazel.build/bazel-release.pub.gpg stable jdk1.8
55
install_packages bazel
66
else
7-
brew install bazel
7+
safe_brew install bazel
88
fi
99
fi
1010
}
@@ -25,7 +25,7 @@ add_grpc_php_plugin_brew() {
2525
. "${0%/*}"/tools/brew.sh
2626
configure_brew
2727
[ -e /usr/local/bin/protoc ] && sudo mv /usr/local/bin/protoc /tmp/protoc && sudo mv /usr/local/include/google /tmp
28-
brew install grpc
28+
safe_brew install grpc
2929
brew link --force --overwrite grpc >/dev/null 2>&1
3030
[ -e /tmp/protoc ] && sudo mv /tmp/protoc /usr/local/bin/protoc && sudo mv /tmp/google /usr/local/include/
3131
grpc_tag="v$(brew info grpc | grep "grpc:" | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")"

0 commit comments

Comments
 (0)