86 lines
2.0 KiB
Go
86 lines
2.0 KiB
Go
|
|
package virtualmachine
|
||
|
|
|
||
|
|
import (
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
// ExponentialBackoff calculates the next retry delay using exponential backoff
|
||
|
|
// Returns delays: 30s, 1m, 2m, 5m, 10m, then caps at 10m
|
||
|
|
func ExponentialBackoff(attempt int) time.Duration {
|
||
|
|
delays := []time.Duration{
|
||
|
|
30 * time.Second, // First retry: 30 seconds
|
||
|
|
1 * time.Minute, // Second retry: 1 minute
|
||
|
|
2 * time.Minute, // Third retry: 2 minutes
|
||
|
|
5 * time.Minute, // Fourth retry: 5 minutes
|
||
|
|
10 * time.Minute, // Fifth and subsequent retries: 10 minutes (capped)
|
||
|
|
}
|
||
|
|
|
||
|
|
if attempt < len(delays) {
|
||
|
|
return delays[attempt]
|
||
|
|
}
|
||
|
|
return delays[len(delays)-1] // Cap at maximum delay
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetRequeueDelay calculates requeue delay based on error type and attempt count
|
||
|
|
func GetRequeueDelay(err error, attemptCount int) time.Duration {
|
||
|
|
if err == nil {
|
||
|
|
return 30 * time.Second // Default for successful operations
|
||
|
|
}
|
||
|
|
|
||
|
|
errStr := err.Error()
|
||
|
|
|
||
|
|
// Non-retryable errors (configuration issues)
|
||
|
|
if containsAny(errStr, []string{
|
||
|
|
"cannot get provider config",
|
||
|
|
"cannot get credentials",
|
||
|
|
"cannot find site",
|
||
|
|
"cannot create Proxmox client",
|
||
|
|
}) {
|
||
|
|
return 2 * time.Minute // Fixed delay for config errors
|
||
|
|
}
|
||
|
|
|
||
|
|
// Quota errors - longer delay
|
||
|
|
if containsAny(errStr, []string{
|
||
|
|
"quota exceeded",
|
||
|
|
"quota check failed",
|
||
|
|
}) {
|
||
|
|
return 5 * time.Minute
|
||
|
|
}
|
||
|
|
|
||
|
|
// VM creation failures - use exponential backoff
|
||
|
|
if containsAny(errStr, []string{
|
||
|
|
"cannot create VM",
|
||
|
|
"failed to import image",
|
||
|
|
"importdisk",
|
||
|
|
"not implemented",
|
||
|
|
}) {
|
||
|
|
return ExponentialBackoff(attemptCount)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Node health issues
|
||
|
|
if containsAny(errStr, []string{
|
||
|
|
"node",
|
||
|
|
"unhealthy",
|
||
|
|
"not reachable",
|
||
|
|
}) {
|
||
|
|
return 2 * time.Minute
|
||
|
|
}
|
||
|
|
|
||
|
|
// Default: exponential backoff
|
||
|
|
return ExponentialBackoff(attemptCount)
|
||
|
|
}
|
||
|
|
|
||
|
|
func containsAny(s string, substrings []string) bool {
|
||
|
|
for _, substr := range substrings {
|
||
|
|
if len(s) >= len(substr) {
|
||
|
|
for i := 0; i <= len(s)-len(substr); i++ {
|
||
|
|
if s[i:i+len(substr)] == substr {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|