Initial commit: Nginx Proxy Manager utility scripts and documentation
- CLAUDE.md: Project guidance for Claude Code - PROJECT_DOCUMENTATION.md: Complete project documentation - upload_log_file_fixed.sh: Fixed rclone upload functions with proper error handling - error_handling_comparison.sh: Documentation of rclone error handling patterns This repository contains utility scripts for managing nginx-proxy-manager log streaming to Cloudflare R2 storage, designed for CrowdSec integration.
This commit is contained in:
kappa
171
upload_log_file_fixed.sh
Normal file
171
upload_log_file_fixed.sh
Normal file
@@ -0,0 +1,171 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Fixed upload_log_file function with proper error handling
|
||||
upload_log_file() {
|
||||
local source_file="$1"
|
||||
local dest_path="$2"
|
||||
local max_retries="${3:-3}"
|
||||
local retry_delay="${4:-5}"
|
||||
|
||||
# Validate input parameters
|
||||
if [[ -z "$source_file" || -z "$dest_path" ]]; then
|
||||
echo "ERROR: Missing required parameters. Usage: upload_log_file <source_file> <dest_path> [max_retries] [retry_delay]" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if source file exists
|
||||
if [[ ! -f "$source_file" ]]; then
|
||||
echo "ERROR: Source file '$source_file' does not exist" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local retry_count=0
|
||||
local upload_success=false
|
||||
|
||||
echo "Starting upload: $source_file -> $dest_path"
|
||||
|
||||
while [[ $retry_count -lt $max_retries ]] && [[ "$upload_success" = false ]]; do
|
||||
if [[ $retry_count -gt 0 ]]; then
|
||||
echo "Retry attempt $retry_count of $max_retries after ${retry_delay}s delay..."
|
||||
sleep "$retry_delay"
|
||||
fi
|
||||
|
||||
echo "Uploading $(basename "$source_file") (attempt $((retry_count + 1))/$max_retries)..."
|
||||
|
||||
# Create temporary files for capturing output
|
||||
local stdout_file=$(mktemp)
|
||||
local stderr_file=$(mktemp)
|
||||
|
||||
# Run rclone copyto and capture both stdout and stderr
|
||||
# The exit code is the most reliable indicator of success/failure
|
||||
if rclone copyto "$source_file" "$dest_path" \
|
||||
--config="${RCLONE_CONFIG:-$HOME/.config/rclone/rclone.conf}" \
|
||||
--progress \
|
||||
--stats=1s \
|
||||
--stats-one-line \
|
||||
--retries=1 \
|
||||
--low-level-retries=1 \
|
||||
--timeout=300s \
|
||||
--contimeout=60s \
|
||||
> "$stdout_file" 2> "$stderr_file"; then
|
||||
|
||||
# rclone exited with code 0 - success
|
||||
upload_success=true
|
||||
echo "✅ Upload successful: $(basename "$source_file")"
|
||||
|
||||
# Optional: Show final transfer stats if available
|
||||
if [[ -s "$stdout_file" ]]; then
|
||||
local last_line=$(tail -n1 "$stdout_file")
|
||||
if [[ "$last_line" =~ Transferred ]]; then
|
||||
echo "📊 $last_line"
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
# rclone exited with non-zero code - failure
|
||||
local exit_code=$?
|
||||
echo "❌ Upload failed with exit code: $exit_code"
|
||||
|
||||
# Show error details
|
||||
if [[ -s "$stderr_file" ]]; then
|
||||
echo "Error details:"
|
||||
cat "$stderr_file" | head -10 # Limit error output
|
||||
fi
|
||||
|
||||
# Show last few lines of stdout for context
|
||||
if [[ -s "$stdout_file" ]]; then
|
||||
echo "Last output:"
|
||||
tail -n3 "$stdout_file"
|
||||
fi
|
||||
|
||||
retry_count=$((retry_count + 1))
|
||||
fi
|
||||
|
||||
# Clean up temporary files
|
||||
rm -f "$stdout_file" "$stderr_file"
|
||||
done
|
||||
|
||||
if [[ "$upload_success" = true ]]; then
|
||||
echo "🎉 Final result: Upload completed successfully"
|
||||
return 0
|
||||
else
|
||||
echo "💥 Final result: Upload failed after $max_retries attempts"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Alternative minimal version focused only on exit code checking
|
||||
upload_log_file_minimal() {
|
||||
local source_file="$1"
|
||||
local dest_path="$2"
|
||||
|
||||
echo "Uploading $(basename "$source_file")..."
|
||||
|
||||
# Simple approach: rely solely on rclone's exit code
|
||||
# Suppress progress output to avoid confusion
|
||||
if rclone copyto "$source_file" "$dest_path" \
|
||||
--config="${RCLONE_CONFIG:-$HOME/.config/rclone/rclone.conf}" \
|
||||
--retries=2 \
|
||||
--timeout=300s \
|
||||
--quiet; then
|
||||
echo "✅ Upload successful: $(basename "$source_file")"
|
||||
return 0
|
||||
else
|
||||
local exit_code=$?
|
||||
echo "❌ Upload failed with exit code: $exit_code"
|
||||
return $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# Example usage function showing proper error handling in monitoring loop
|
||||
monitor_and_upload_logs() {
|
||||
local log_directory="$1"
|
||||
local remote_path="$2"
|
||||
|
||||
while true; do
|
||||
# Find log files to upload
|
||||
local files_to_upload=($(find "$log_directory" -name "*.log" -type f -mmin +1))
|
||||
|
||||
for log_file in "${files_to_upload[@]}"; do
|
||||
local filename=$(basename "$log_file")
|
||||
local dest_path="$remote_path/$filename"
|
||||
|
||||
# Use the fixed upload function
|
||||
if upload_log_file "$log_file" "$dest_path"; then
|
||||
echo "Successfully uploaded $filename, moving to processed/"
|
||||
mkdir -p "$log_directory/processed"
|
||||
mv "$log_file" "$log_directory/processed/"
|
||||
else
|
||||
echo "Failed to upload $filename, will retry next cycle"
|
||||
# Don't exit the monitoring loop on upload failure
|
||||
# The file will be retried in the next cycle
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait before next check
|
||||
sleep 60
|
||||
done
|
||||
}
|
||||
|
||||
# Test function to validate rclone configuration
|
||||
test_rclone_upload() {
|
||||
local test_file=$(mktemp)
|
||||
echo "test upload $(date)" > "$test_file"
|
||||
|
||||
echo "Testing rclone upload functionality..."
|
||||
if upload_log_file "$test_file" "r2:your-bucket/test/test_$(date +%s).txt"; then
|
||||
echo "✅ rclone upload test passed"
|
||||
rm -f "$test_file"
|
||||
return 0
|
||||
else
|
||||
echo "❌ rclone upload test failed"
|
||||
rm -f "$test_file"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# If script is run directly, run test
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
echo "Running rclone upload test..."
|
||||
test_rclone_upload
|
||||
fi
|
||||
Reference in New Issue
Block a user