9daf1fd378
- Add comprehensive database migrations (001-024) for schema evolution - Enhance API schema with expanded type definitions and resolvers - Add new middleware: audit logging, rate limiting, MFA enforcement, security, tenant auth - Implement new services: AI optimization, billing, blockchain, compliance, marketplace - Add adapter layer for cloud integrations (Cloudflare, Kubernetes, Proxmox, storage) - Update Crossplane provider with enhanced VM management capabilities - Add comprehensive test suite for API endpoints and services - Update frontend components with improved GraphQL subscriptions and real-time updates - Enhance security configurations and headers (CSP, CORS, etc.) - Update documentation and configuration files - Add new CI/CD workflows and validation scripts - Implement design system improvements and UI enhancements
86 lines
2.0 KiB
Go
86 lines
2.0 KiB
Go
package virtualmachine
|
|
|
|
import (
|
|
"time"
|
|
)
|
|
|
|
// ExponentialBackoff calculates the next retry delay using exponential backoff
|
|
// Returns delays: 30s, 1m, 2m, 5m, 10m, then caps at 10m
|
|
func ExponentialBackoff(attempt int) time.Duration {
|
|
delays := []time.Duration{
|
|
30 * time.Second, // First retry: 30 seconds
|
|
1 * time.Minute, // Second retry: 1 minute
|
|
2 * time.Minute, // Third retry: 2 minutes
|
|
5 * time.Minute, // Fourth retry: 5 minutes
|
|
10 * time.Minute, // Fifth and subsequent retries: 10 minutes (capped)
|
|
}
|
|
|
|
if attempt < len(delays) {
|
|
return delays[attempt]
|
|
}
|
|
return delays[len(delays)-1] // Cap at maximum delay
|
|
}
|
|
|
|
// GetRequeueDelay calculates requeue delay based on error type and attempt count
|
|
func GetRequeueDelay(err error, attemptCount int) time.Duration {
|
|
if err == nil {
|
|
return 30 * time.Second // Default for successful operations
|
|
}
|
|
|
|
errStr := err.Error()
|
|
|
|
// Non-retryable errors (configuration issues)
|
|
if containsAny(errStr, []string{
|
|
"cannot get provider config",
|
|
"cannot get credentials",
|
|
"cannot find site",
|
|
"cannot create Proxmox client",
|
|
}) {
|
|
return 2 * time.Minute // Fixed delay for config errors
|
|
}
|
|
|
|
// Quota errors - longer delay
|
|
if containsAny(errStr, []string{
|
|
"quota exceeded",
|
|
"quota check failed",
|
|
}) {
|
|
return 5 * time.Minute
|
|
}
|
|
|
|
// VM creation failures - use exponential backoff
|
|
if containsAny(errStr, []string{
|
|
"cannot create VM",
|
|
"failed to import image",
|
|
"importdisk",
|
|
"not implemented",
|
|
}) {
|
|
return ExponentialBackoff(attemptCount)
|
|
}
|
|
|
|
// Node health issues
|
|
if containsAny(errStr, []string{
|
|
"node",
|
|
"unhealthy",
|
|
"not reachable",
|
|
}) {
|
|
return 2 * time.Minute
|
|
}
|
|
|
|
// Default: exponential backoff
|
|
return ExponentialBackoff(attemptCount)
|
|
}
|
|
|
|
func containsAny(s string, substrings []string) bool {
|
|
for _, substr := range substrings {
|
|
if len(s) >= len(substr) {
|
|
for i := 0; i <= len(s)-len(substr); i++ {
|
|
if s[i:i+len(substr)] == substr {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|