Skip to content

fix: support serverless logs and robust timeout in diagnose scripts #5671

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Ayush-Patel-56 wants to merge 1 commit into
base: master
Choose a base branch
from
+185 −5
Open

fix: support serverless logs and robust timeout in diagnose scripts #5671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion tools/diagnose-fluid-alluxio.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,16 @@ print_usage() {
run() {
echo
echo "-----------------run $*------------------"
timeout 10s "$@"
if command -v timeout >/dev/null 2>&1; then
timeout 10s "$@"
elif command -v gtimeout >/dev/null 2>&1; then
gtimeout 10s "$@"
elif command -v perl >/dev/null 2>&1; then
# Use Perl to enforce timeout on systems without GNU coreutils (like standard macOS)
perl -e 'alarm shift; exec @ARGV' 10 "$@"
else
"$@"
fi
if [ $? != 0 ]; then
echo "failed to collect info: $*"
fi
Expand Down Expand Up @@ -57,6 +66,41 @@ runtime_pod_logs() {
core_component "${runtime_namespace}" "alluxio-fuse" "role=alluxio-fuse" "release=${runtime_name}"
}

serverless_pod_logs() {
# Check if kubectl is available
if ! command -v kubectl >/dev/null 2>&1; then
echo "kubectl not found, skipping serverless pod logs collection"
return
fi

local namespace="${runtime_namespace}"
# More specific selector: injected and matching the specific dataset/runtime
local label_selector="serverless.fluid.io/inject=true,fluid.io/dataset=${runtime_namespace}.${runtime_name}"

# Get all pods with the serverless inject label
local pods
pods=$(kubectl get po -n "${namespace}" -l "${label_selector}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
if [[ $? -ne 0 ]]; then
echo "failed to get serverless pods in namespace ${namespace}"
return
fi

if [[ -n "$pods" ]]; then
mkdir -p "$diagnose_dir/pods-${namespace}-serverless"
for po in ${pods}; do
# Find all containers containing fluid-fuse (covers init-fluid-fuse and fluid-fuse)
local containers
containers=$(kubectl get po "${po}" -n "${namespace}" -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' 2>/dev/null | tr ' ' '\n' | grep 'fluid-fuse')
if [[ -n "$containers" ]]; then
for container in ${containers}; do
kubectl logs "${po}" -c "${container}" -n "${namespace}" &>"$diagnose_dir/pods-${namespace}-serverless/${po}-${container}.log" 2>&1
done
fi
done
fi
return
}

core_component() {
# namespace container selectors...
local namespace="$1"
Expand Down Expand Up @@ -105,6 +149,7 @@ pd_collect() {
pod_status "${fluid_namespace}"
pod_status "${runtime_namespace}"
runtime_pod_logs
serverless_pod_logs
fluid_pod_logs
kubectl_resource
archive
Expand Down
49 changes: 47 additions & 2 deletions tools/diagnose-fluid-goosefs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,16 @@ print_usage() {
run() {
echo
echo "-----------------run $*------------------"
timeout 10s "$@"
if command -v timeout >/dev/null 2>&1; then
timeout 10s "$@"
elif command -v gtimeout >/dev/null 2>&1; then
gtimeout 10s "$@"
elif command -v perl >/dev/null 2>&1; then
# Use Perl to enforce timeout on systems without GNU coreutils (like standard macOS)
perl -e 'alarm shift; exec @ARGV' 10 "$@"
else
"$@"
fi
if [ $? != 0 ]; then
echo "failed to collect info: $*"
fi
Expand Down Expand Up @@ -50,6 +59,41 @@ runtime_pod_logs() {
core_component "${runtime_namespace}" "goosefs-fuse" "role=goosefs-fuse" "release=${runtime_name}"
}

serverless_pod_logs() {
# Check if kubectl is available
if ! command -v kubectl >/dev/null 2>&1; then
echo "kubectl not found, skipping serverless pod logs collection"
return
fi

local namespace="${runtime_namespace}"
# More specific selector: injected and matching the specific dataset/runtime
local label_selector="serverless.fluid.io/inject=true,fluid.io/dataset=${runtime_namespace}.${runtime_name}"

# Get all pods with the serverless inject label
local pods
pods=$(kubectl get po -n "${namespace}" -l "${label_selector}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
if [[ $? -ne 0 ]]; then
echo "failed to get serverless pods in namespace ${namespace}"
return
fi

if [[ -n "$pods" ]]; then
mkdir -p "$diagnose_dir/pods-${namespace}-serverless"
for po in ${pods}; do
# Find all containers containing fluid-fuse (covers init-fluid-fuse and fluid-fuse)
local containers
containers=$(kubectl get po "${po}" -n "${namespace}" -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' 2>/dev/null | tr ' ' '\n' | grep 'fluid-fuse')
if [[ -n "$containers" ]]; then
for container in ${containers}; do
kubectl logs "${po}" -c "${container}" -n "${namespace}" &>"$diagnose_dir/pods-${namespace}-serverless/${po}-${container}.log" 2>&1
done
fi
done
fi
return
}

core_component() {
# namespace container selectors...
local namespace="$1"
Expand All @@ -65,7 +109,7 @@ core_component() {
mkdir -p "$diagnose_dir/pods-${namespace}"
pods=$(kubectl get po -n ${namespace} "${constrains}" | awk '{print $1}' | grep -v NAME)
for po in ${pods}; do
if [[ "${namespace}"="${fluid_namesapce}" ]]; then
if [[ "${namespace}" == "${fluid_namespace}" ]]; then
kubectl logs "${po}" -c "$container" -n ${namespace} &>"$diagnose_dir/pods-${namespace}/${po}-${container}.log" 2>&1
else
kubectl cp "${namespace}/${po}":/opt/goosefs/logs -c "${container}" "$diagnose_dir/pods-${namespace}/${po}-${container}" 2>&1
Expand Down Expand Up @@ -93,6 +137,7 @@ pd_collect() {
pod_status "${fluid_namespace}"
pod_status "${runtime_namespace}"
runtime_pod_logs
serverless_pod_logs
fluid_pod_logs
kubectl_resource
archive
Expand Down
47 changes: 46 additions & 1 deletion tools/diagnose-fluid-jindo.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,16 @@ print_usage() {
run() {
echo
echo "-----------------run $*------------------"
timeout 10s "$@"
if command -v timeout >/dev/null 2>&1; then
timeout 10s "$@"
elif command -v gtimeout >/dev/null 2>&1; then
gtimeout 10s "$@"
elif command -v perl >/dev/null 2>&1; then
# Use Perl to enforce timeout on systems without GNU coreutils (like standard macOS)
perl -e 'alarm shift; exec @ARGV' 10 "$@"
else
"$@"
fi
if [ $? != 0 ]; then
echo "failed to collect info: $*"
fi
Expand Down Expand Up @@ -55,6 +64,41 @@ runtime_pod_logs() {
core_component "${runtime_namespace}" "jindofs-fuse" "role=jindofs-fuse" "release=${runtime_name}"
}

serverless_pod_logs() {
# Check if kubectl is available
if ! command -v kubectl >/dev/null 2>&1; then
echo "kubectl not found, skipping serverless pod logs collection"
return
fi

local namespace="${runtime_namespace}"
# More specific selector: injected and matching the specific dataset/runtime
local label_selector="serverless.fluid.io/inject=true,fluid.io/dataset=${runtime_namespace}.${runtime_name}"

# Get all pods with the serverless inject label
local pods
pods=$(kubectl get po -n "${namespace}" -l "${label_selector}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
if [[ $? -ne 0 ]]; then
echo "failed to get serverless pods in namespace ${namespace}"
return
fi

if [[ -n "$pods" ]]; then
mkdir -p "$diagnose_dir/pods-${namespace}-serverless"
for po in ${pods}; do
# Find all containers containing fluid-fuse (covers init-fluid-fuse and fluid-fuse)
local containers
containers=$(kubectl get po "${po}" -n "${namespace}" -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' 2>/dev/null | tr ' ' '\n' | grep 'fluid-fuse')
if [[ -n "$containers" ]]; then
for container in ${containers}; do
kubectl logs "${po}" -c "${container}" -n "${namespace}" &>"$diagnose_dir/pods-${namespace}-serverless/${po}-${container}.log" 2>&1
done
fi
done
fi
return
}

core_component() {
# namespace container selectors...
local namespace="$1"
Expand Down Expand Up @@ -99,6 +143,7 @@ pd_collect() {
pod_status "${fluid_namespace}"
pod_status "${runtime_namespace}"
runtime_pod_logs
serverless_pod_logs
fluid_pod_logs
kubectl_resource
archive
Expand Down
47 changes: 46 additions & 1 deletion tools/diagnose-fluid-juicefs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,16 @@ print_usage() {
run() {
echo
echo "-----------------run $*------------------"
timeout 10s "$@"
if command -v timeout >/dev/null 2>&1; then
timeout 10s "$@"
elif command -v gtimeout >/dev/null 2>&1; then
gtimeout 10s "$@"
elif command -v perl >/dev/null 2>&1; then
# Use Perl to enforce timeout on systems without GNU coreutils (like standard macOS)
perl -e 'alarm shift; exec @ARGV' 10 "$@"
else
"$@"
fi
if [ $? != 0 ]; then
echo "failed to collect info: $*"
fi
Expand Down Expand Up @@ -54,6 +63,41 @@ runtime_pod_logs() {
core_component "${runtime_namespace}" "juicefs-fuse" "role=juicefs-fuse" "release=${runtime_name}"
}

serverless_pod_logs() {
# Check if kubectl is available
if ! command -v kubectl >/dev/null 2>&1; then
echo "kubectl not found, skipping serverless pod logs collection"
return
fi

local namespace="${runtime_namespace}"
# More specific selector: injected and matching the specific dataset/runtime
local label_selector="serverless.fluid.io/inject=true,fluid.io/dataset=${runtime_namespace}.${runtime_name}"

# Get all pods with the serverless inject label
local pods
pods=$(kubectl get po -n "${namespace}" -l "${label_selector}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
if [[ $? -ne 0 ]]; then
echo "failed to get serverless pods in namespace ${namespace}"
return
fi

if [[ -n "$pods" ]]; then
mkdir -p "$diagnose_dir/pods-${namespace}-serverless"
for po in ${pods}; do
# Find all containers containing fluid-fuse (covers init-fluid-fuse and fluid-fuse)
local containers
containers=$(kubectl get po "${po}" -n "${namespace}" -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' 2>/dev/null | tr ' ' '\n' | grep 'fluid-fuse')
if [[ -n "$containers" ]]; then
for container in ${containers}; do
kubectl logs "${po}" -c "${container}" -n "${namespace}" &>"$diagnose_dir/pods-${namespace}-serverless/${po}-${container}.log" 2>&1
done
fi
done
fi
return
}

core_component() {
# namespace container selectors...
local namespace="$1"
Expand Down Expand Up @@ -98,6 +142,7 @@ pd_collect() {
pod_status "${fluid_namespace}"
pod_status "${runtime_namespace}"
runtime_pod_logs
serverless_pod_logs
fluid_pod_logs
kubectl_resource
archive
Expand Down