fix: Improve consistency and add rollback support
1. haproxy_add_servers - disk-first pattern - Save ALL servers to config FIRST - Then update HAProxy - Rollback all on unexpected error - Rollback failed slots individually 2. remove_domain_from_config - file locking - Add fcntl.LOCK_EX for consistency with other config ops - Prevents race conditions during concurrent access 3. haproxy_add_domain - rollback on HAProxy failure - Wrap HAProxy map update in try/except - Rollback map file if HAProxy command fails - Rollback server config if server setup fails Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -561,15 +561,21 @@ def remove_server_from_config(domain: str, slot: int) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def remove_domain_from_config(domain: str) -> None:
|
def remove_domain_from_config(domain: str) -> None:
|
||||||
"""Remove all server configurations for a domain.
|
"""Remove domain from persistent config with file locking.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
domain: Domain name to remove
|
domain: Domain name to remove
|
||||||
"""
|
"""
|
||||||
|
lock_path = f"{SERVERS_FILE}.lock"
|
||||||
|
with open(lock_path, 'w') as lock_file:
|
||||||
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
config = load_servers_config()
|
config = load_servers_config()
|
||||||
if domain in config:
|
if domain in config:
|
||||||
del config[domain]
|
del config[domain]
|
||||||
save_servers_config(config)
|
save_servers_config(config)
|
||||||
|
finally:
|
||||||
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
def get_server_suffixes(http_port: int) -> List[Tuple[str, int]]:
|
def get_server_suffixes(http_port: int) -> List[Tuple[str, int]]:
|
||||||
@@ -788,28 +794,43 @@ def haproxy_add_domain(domain: str, ip: str = "", http_port: int = 80) -> str:
|
|||||||
entries = get_map_contents()
|
entries = get_map_contents()
|
||||||
entries.append((domain, pool))
|
entries.append((domain, pool))
|
||||||
entries.append((f".{domain}", pool))
|
entries.append((f".{domain}", pool))
|
||||||
|
try:
|
||||||
save_map_file(entries)
|
save_map_file(entries)
|
||||||
|
except IOError as e:
|
||||||
|
return f"Error: Failed to save map file: {e}"
|
||||||
|
|
||||||
# Update HAProxy map via Runtime API (immediate effect)
|
# Then update HAProxy map via Runtime API
|
||||||
|
try:
|
||||||
haproxy_cmd(f"add map {MAP_FILE_CONTAINER} {domain} {pool}")
|
haproxy_cmd(f"add map {MAP_FILE_CONTAINER} {domain} {pool}")
|
||||||
haproxy_cmd(f"add map {MAP_FILE_CONTAINER} .{domain} {pool}")
|
haproxy_cmd(f"add map {MAP_FILE_CONTAINER} .{domain} {pool}")
|
||||||
|
except HaproxyError as e:
|
||||||
|
# Rollback: remove the domain we just added from entries and re-save
|
||||||
|
rollback_entries = [(d, b) for d, b in entries if d != domain and d != f".{domain}"]
|
||||||
|
try:
|
||||||
|
save_map_file(rollback_entries)
|
||||||
|
except IOError:
|
||||||
|
logger.error("Failed to rollback map file after HAProxy error")
|
||||||
|
return f"Error: Failed to update HAProxy map: {e}"
|
||||||
|
|
||||||
# If IP provided, add server to slot 1
|
# If IP provided, add server to slot 1
|
||||||
if ip:
|
if ip:
|
||||||
# Save server config to disk first
|
# Save server config to disk first
|
||||||
add_server_to_config(domain, 1, ip, http_port)
|
add_server_to_config(domain, 1, ip, http_port)
|
||||||
|
|
||||||
|
try:
|
||||||
for suffix, port in get_server_suffixes(http_port):
|
for suffix, port in get_server_suffixes(http_port):
|
||||||
server = f"{pool}{suffix}_1"
|
server = f"{pool}{suffix}_1"
|
||||||
haproxy_cmd(f"set server {pool}/{server} addr {ip} port {port}")
|
haproxy_cmd(f"set server {pool}/{server} addr {ip} port {port}")
|
||||||
haproxy_cmd(f"set server {pool}/{server} state ready")
|
haproxy_cmd(f"set server {pool}/{server} state ready")
|
||||||
|
except HaproxyError as e:
|
||||||
|
# Rollback server config on failure
|
||||||
|
remove_server_from_config(domain, 1)
|
||||||
|
return f"Domain {domain} added to {pool} but server config failed: {e}"
|
||||||
|
|
||||||
return f"Domain {domain} added to {pool} with server {ip}:{http_port}"
|
return f"Domain {domain} added to {pool} with server {ip}:{http_port}"
|
||||||
|
|
||||||
return f"Domain {domain} added to {pool} (no servers configured)"
|
return f"Domain {domain} added to {pool} (no servers configured)"
|
||||||
|
|
||||||
except IOError as e:
|
|
||||||
return f"Error: Failed to update map file: {e}"
|
|
||||||
except HaproxyError as e:
|
except HaproxyError as e:
|
||||||
return f"Error: {e}"
|
return f"Error: {e}"
|
||||||
|
|
||||||
@@ -1072,22 +1093,38 @@ def haproxy_add_servers(domain: str, servers: str) -> str:
|
|||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
return f"Error: {e}"
|
return f"Error: {e}"
|
||||||
|
|
||||||
# Add all servers
|
# Save ALL servers to config FIRST (disk-first pattern)
|
||||||
|
for server_config in validated_servers:
|
||||||
|
slot = server_config["slot"]
|
||||||
|
ip = server_config["ip"]
|
||||||
|
http_port = server_config["http_port"]
|
||||||
|
add_server_to_config(domain, slot, ip, http_port)
|
||||||
|
|
||||||
|
# Then update HAProxy
|
||||||
added = []
|
added = []
|
||||||
errors = []
|
errors = []
|
||||||
|
failed_slots = []
|
||||||
for srv in validated_servers:
|
|
||||||
slot = srv["slot"]
|
|
||||||
ip = srv["ip"]
|
|
||||||
http_port = srv["http_port"]
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
server = configure_server_slot(backend, server_prefix, slot, ip, http_port)
|
for server_config in validated_servers:
|
||||||
# Save to persistent config
|
slot = server_config["slot"]
|
||||||
add_server_to_config(domain, slot, ip, http_port)
|
ip = server_config["ip"]
|
||||||
|
http_port = server_config["http_port"]
|
||||||
|
try:
|
||||||
|
configure_server_slot(backend, server_prefix, slot, ip, http_port)
|
||||||
added.append(f"slot {slot}: {ip}:{http_port}")
|
added.append(f"slot {slot}: {ip}:{http_port}")
|
||||||
except (HaproxyError, IOError) as e:
|
except HaproxyError as e:
|
||||||
|
failed_slots.append(slot)
|
||||||
errors.append(f"slot {slot}: {e}")
|
errors.append(f"slot {slot}: {e}")
|
||||||
|
except Exception as e:
|
||||||
|
# Rollback all saved configs on unexpected error
|
||||||
|
for server_config in validated_servers:
|
||||||
|
remove_server_from_config(domain, server_config["slot"])
|
||||||
|
return f"Error: {e}"
|
||||||
|
|
||||||
|
# Rollback failed slots from config
|
||||||
|
for slot in failed_slots:
|
||||||
|
remove_server_from_config(domain, slot)
|
||||||
|
|
||||||
# Build result message
|
# Build result message
|
||||||
result_parts = []
|
result_parts = []
|
||||||
|
|||||||
Reference in New Issue
Block a user