diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..528b121 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "virtual-banker"] + path = virtual-banker + url = https://gitea.d-bis.org/d-bis/virtual-banker.git diff --git a/virtual-banker b/virtual-banker new file mode 160000 index 0000000..b4753ce --- /dev/null +++ b/virtual-banker @@ -0,0 +1 @@ +Subproject commit b4753cef7e78960337bb6ed9e4865fac03fb33d8 diff --git a/virtual-banker/COMPLETE_TASK_LIST.md b/virtual-banker/COMPLETE_TASK_LIST.md deleted file mode 100644 index f2e1806..0000000 --- a/virtual-banker/COMPLETE_TASK_LIST.md +++ /dev/null @@ -1,860 +0,0 @@ -# Virtual Banker - Complete Task, Recommendation, and Suggestion List - -**Last Updated**: 2025-01-20 -**Status**: Implementation Complete, Production Integration Pending - ---- - -## Table of Contents - -1. [Completed Tasks](#completed-tasks) -2. [Critical Tasks (Must Do)](#critical-tasks-must-do) -3. [High Priority Tasks](#high-priority-tasks) -4. [Medium Priority Tasks](#medium-priority-tasks) -5. [Low Priority Tasks](#low-priority-tasks) -6. [Recommendations](#recommendations) -7. [Suggestions for Enhancement](#suggestions-for-enhancement) -8. [Testing Tasks](#testing-tasks) -9. [Documentation Tasks](#documentation-tasks) -10. [Production Readiness Checklist](#production-readiness-checklist) - ---- - -## Completed Tasks ✅ - -### Phase 0: Foundation & Widget -- [x] Backend directory structure created -- [x] Session service with JWT validation -- [x] REST API endpoints (create, refresh, end session) -- [x] Database migrations (sessions, tenants, conversations, knowledge base, user profiles) -- [x] Redis integration for session caching -- [x] Embeddable React/TypeScript widget -- [x] Chat UI components (ChatPanel, VoiceControls, AvatarView, Captions, Settings) -- [x] Widget loader script (`widget.js`) -- [x] PostMessage API for host integration -- [x] Accessibility features (ARIA, keyboard navigation, captions) -- [x] Theming system -- [x] Docker Compose integration - -### Phase 1: Voice & Realtime -- [x] WebRTC gateway infrastructure -- [x] WebSocket signaling support -- [x] ASR service interface and mock implementation -- [x] TTS service interface and mock implementation -- [x] Conversation orchestrator with state machine -- [x] Barge-in support (interrupt handling) -- [x] Audio/video synchronization framework - -### Phase 2: LLM & RAG -- [x] LLM gateway interface and mock -- [x] Multi-tenant prompt builder -- [x] RAG service with pgvector -- [x] Document ingestion pipeline -- [x] Vector similarity search -- [x] Tool framework (registry, executor, audit logging) -- [x] Banking tool integrations: - - [x] get_account_status - - [x] create_support_ticket - - [x] schedule_appointment - - [x] submit_payment -- [x] Banking service HTTP client -- [x] Fallback mechanisms for service unavailability - -### Phase 3: Avatar System -- [x] Unreal Engine setup documentation -- [x] Renderer service structure -- [x] PixelStreaming integration framework -- [x] Animation controller: - - [x] Viseme mapping (phoneme → viseme) - - [x] Expression system (valence/arousal → facial expressions) - - [x] Gesture system (rule-based gesture selection) - -### Phase 4: Memory & Observability -- [x] Memory service (user profiles, conversation history) -- [x] Observability (tracing, metrics) -- [x] Safety/compliance (content filtering, rate limiting) -- [x] PII redaction framework - -### Phase 5: Enterprise Features -- [x] Multi-tenancy support -- [x] Tenant configuration system -- [x] Complete documentation - -### Integration Tasks -- [x] Orchestrator connected to all services -- [x] Banking tools connected to backend services -- [x] WebSocket support added to API -- [x] Startup scripts created -- [x] All compilation errors fixed -- [x] Code builds successfully - ---- - -## Critical Tasks (Must Do) - -### 1. Replace Mock Services with Real APIs - -#### ASR Service Integration -- [ ] **Get API credentials**: - - [ ] Sign up for Deepgram account OR - - [ ] Set up Google Cloud Speech-to-Text - - [ ] Obtain API keys and configure environment variables - -- [ ] **Implement Deepgram Integration**: - - [ ] Update `backend/asr/service.go` - - [ ] Implement WebSocket streaming connection - - [ ] Handle partial and final transcripts - - [ ] Extract word-level timestamps for lip sync - - [ ] Add error handling and retry logic - - [ ] Test with real audio streams - -- [ ] **OR Implement Google STT**: - - [ ] Set up Google Cloud credentials - - [ ] Implement streaming recognition - - [ ] Handle language detection - - [ ] Add punctuation and formatting - -#### TTS Service Integration -- [ ] **Get API credentials**: - - [ ] Sign up for ElevenLabs account OR - - [ ] Set up Azure Cognitive Services TTS - - [ ] Obtain API keys - -- [ ] **Implement ElevenLabs Integration**: - - [ ] Update `backend/tts/service.go` - - [ ] Implement streaming synthesis - - [ ] Configure voice selection per tenant - - [ ] Extract phoneme/viseme timings - - [ ] Add SSML support - - [ ] Test voice quality and latency - -- [ ] **OR Implement Azure TTS**: - - [ ] Set up Azure credentials - - [ ] Implement neural voice synthesis - - [ ] Configure SSML - - [ ] Add voice cloning if needed - -#### LLM Gateway Integration -- [ ] **Get API credentials**: - - [ ] Sign up for OpenAI account OR - - [ ] Sign up for Anthropic Claude - - [ ] Obtain API keys - -- [ ] **Implement OpenAI Integration**: - - [ ] Update `backend/llm/gateway.go` - - [ ] Implement function calling - - [ ] Add streaming support - - [ ] Configure model selection (GPT-4, GPT-3.5) - - [ ] Implement output schema enforcement - - [ ] Add emotion/gesture extraction - - [ ] Test with real conversations - -- [ ] **OR Implement Anthropic Claude**: - - [ ] Implement tool use - - [ ] Add streaming - - [ ] Configure model (Claude 3 Opus/Sonnet) - -### 2. Complete WebRTC Implementation - -- [ ] **Implement SDP Offer/Answer Exchange**: - - [ ] Handle SDP offer from client - - [ ] Generate SDP answer - - [ ] Exchange via WebSocket signaling - - [ ] Test connection establishment - -- [ ] **Implement ICE Candidate Handling**: - - [ ] Collect ICE candidates from client - - [ ] Send server ICE candidates - - [ ] Handle candidate exchange - - [ ] Test with various network conditions - -- [ ] **Configure TURN Server**: - - [ ] Set up TURN server (coturn or similar) - - [ ] Configure credentials - - [ ] Add TURN URLs to ICE configuration - - [ ] Test behind NAT/firewall - -- [ ] **Implement Media Streaming**: - - [ ] Stream audio from client → ASR service - - [ ] Stream audio from TTS → client - - [ ] Stream video from avatar → client - - [ ] Synchronize audio/video - - [ ] Handle network issues and reconnection - -### 3. Unreal Engine Avatar Setup - -- [ ] **Install and Configure Unreal Engine**: - - [ ] Download Unreal Engine 5.3+ (or 5.4+) - - [ ] Install on development machine - - [ ] Enable PixelStreaming plugin - - [ ] Configure project settings - -- [ ] **Create/Import Digital Human**: - - [ ] Option A: Use Ready Player Me - - [ ] Install Ready Player Me plugin - - [ ] Generate or import character - - [ ] Configure blendshapes - - [ ] Option B: Use MetaHuman Creator - - [ ] Create MetaHuman character - - [ ] Export to project - - [ ] Configure animation - - [ ] Option C: Import custom character - - [ ] Import FBX/glTF with blendshapes - - [ ] Set up rigging - - [ ] Configure viseme blendshapes - -- [ ] **Set Up Animation System**: - - [ ] Create Animation Blueprint - - [ ] Set up state machine (idle, speaking, gesturing) - - [ ] Connect viseme blendshapes - - [ ] Configure expression blendshapes - - [ ] Add gesture animations - - [ ] Set up idle animations - -- [ ] **Configure PixelStreaming**: - - [ ] Enable PixelStreaming in project settings - - [ ] Configure WebRTC ports - - [ ] Set up signaling server - - [ ] Test streaming locally - -- [ ] **Create Control Blueprint**: - - [ ] Create Blueprint Actor for avatar control - - [ ] Add functions: - - [ ] SetVisemes(VisemeData) - - [ ] SetExpression(Valence, Arousal) - - [ ] SetGesture(GestureType) - - [ ] SetGaze(Target) - - [ ] Connect to renderer service - -- [ ] **Package for Deployment**: - - [ ] Package project for Linux - - [ ] Test on target server - - [ ] Configure GPU requirements - - [ ] Set up instance management - -### 4. Connect to Production Banking Services - -- [ ] **Identify Banking API Endpoints**: - - [ ] Review `backend/banking/` structure - - [ ] Document actual API endpoints - - [ ] Identify authentication requirements - - [ ] Check rate limits and quotas - -- [ ] **Update Banking Client**: - - [ ] Update `backend/tools/banking/integration.go` - - [ ] Match actual endpoint paths - - [ ] Implement proper authentication - - [ ] Add request/response validation - - [ ] Handle errors appropriately - -- [ ] **Test Banking Integrations**: - - [ ] Test account status retrieval - - [ ] Test ticket creation - - [ ] Test appointment scheduling - - [ ] Test payment submission (with proper safeguards) - - [ ] Verify audit logging - ---- - -## High Priority Tasks - -### 5. Testing Infrastructure - -- [ ] **Unit Tests**: - - [ ] Session service tests - - [ ] Orchestrator tests - - [ ] LLM gateway tests - - [ ] RAG service tests - - [ ] Tool executor tests - - [ ] Banking tool tests - - [ ] Safety filter tests - - [ ] Rate limiter tests - -- [ ] **Integration Tests**: - - [ ] API endpoint tests - - [ ] WebSocket connection tests - - [ ] Database integration tests - - [ ] Redis integration tests - - [ ] End-to-end conversation flow tests - -- [ ] **E2E Tests**: - - [ ] Widget initialization - - [ ] Session creation flow - - [ ] Text conversation flow - - [ ] Voice conversation flow (when WebRTC ready) - - [ ] Tool execution flow - - [ ] Error handling scenarios - -- [ ] **Load Testing**: - - [ ] Concurrent session handling - - [ ] API rate limiting - - [ ] Database connection pooling - - [ ] Redis performance - - [ ] Avatar renderer scaling - -### 6. Security Hardening - -- [ ] **Authentication & Authorization**: - - [ ] Implement proper JWT validation - - [ ] Add tenant-specific JWK support - - [ ] Implement role-based access control - - [ ] Add session token rotation - - [ ] Implement CSRF protection - -- [ ] **Input Validation**: - - [ ] Validate all API inputs - - [ ] Sanitize user messages - - [ ] Validate tool parameters - - [ ] Add request size limits - - [ ] Implement SQL injection prevention - -- [ ] **Secrets Management**: - - [ ] Set up secrets management (Vault, AWS Secrets Manager) - - [ ] Remove hardcoded credentials - - [ ] Rotate API keys regularly - - [ ] Encrypt sensitive data at rest - - [ ] Use TLS for all external communication - -- [ ] **Content Security**: - - [ ] Enhance content filtering - - [ ] Add ML-based abuse detection - - [ ] Implement PII detection and redaction - - [ ] Add data loss prevention - - [ ] Monitor for suspicious activity - -### 7. Monitoring & Observability - -- [ ] **Metrics Collection**: - - [ ] Set up Prometheus metrics - - [ ] Add Grafana dashboards - - [ ] Monitor key metrics: - - [ ] Session creation rate - - [ ] Active sessions - - [ ] API latency (p50, p95, p99) - - [ ] Error rates - - [ ] ASR/TTS/LLM latency - - [ ] Tool execution times - - [ ] Avatar render queue depth - -- [ ] **Logging**: - - [ ] Set up centralized logging (ELK, Loki) - - [ ] Implement structured logging (JSON) - - [ ] Add correlation IDs - - [ ] Configure log levels - - [ ] Set up log retention policies - - [ ] Implement log rotation - -- [ ] **Tracing**: - - [ ] Set up OpenTelemetry - - [ ] Add distributed tracing - - [ ] Trace conversation flows - - [ ] Trace tool executions - - [ ] Add performance profiling - -- [ ] **Alerting**: - - [ ] Set up alert rules - - [ ] Configure notification channels - - [ ] Add alerts for: - - [ ] High error rates - - [ ] Service downtime - - [ ] High latency - - [ ] Resource exhaustion - - [ ] Security incidents - -### 8. Performance Optimization - -- [ ] **Database Optimization**: - - [ ] Add database indexes - - [ ] Optimize queries - - [ ] Set up connection pooling - - [ ] Configure read replicas - - [ ] Implement query caching - - [ ] Add database monitoring - -- [ ] **Caching Strategy**: - - [ ] Cache tenant configurations - - [ ] Cache RAG embeddings - - [ ] Cache LLM responses (where appropriate) - - [ ] Cache user profiles - - [ ] Implement cache invalidation - -- [ ] **API Optimization**: - - [ ] Add response compression - - [ ] Implement pagination - - [ ] Add request batching - - [ ] Optimize JSON serialization - - [ ] Add API response caching - -- [ ] **Avatar Rendering Optimization**: - - [ ] Optimize Unreal rendering settings - - [ ] Implement instance pooling - - [ ] Add GPU resource management - - [ ] Optimize video encoding - - [ ] Reduce bandwidth usage - ---- - -## Medium Priority Tasks - -### 9. Enhanced Features - -- [ ] **Multi-language Support**: - - [ ] Add language detection - - [ ] Configure ASR for multiple languages - - [ ] Configure TTS for multiple languages - - [ ] Add translation support - - [ ] Update RAG for multi-language - -- [ ] **Advanced RAG**: - - [ ] Implement reranking (cross-encoder) - - [ ] Add hybrid search (keyword + vector) - - [ ] Implement query expansion - - [ ] Add citation tracking - - [ ] Implement knowledge graph - -- [ ] **Enhanced Tool Framework**: - - [ ] Add tool versioning - - [ ] Implement tool chaining - - [ ] Add conditional tool execution - - [ ] Implement tool result caching - - [ ] Add tool usage analytics - -- [ ] **Conversation Features**: - - [ ] Add conversation summarization - - [ ] Implement context window management - - [ ] Add conversation branching - - [ ] Implement conversation templates - - [ ] Add conversation analytics - -### 10. User Experience Enhancements - -- [ ] **Widget Enhancements**: - - [ ] Add typing indicators - - [ ] Add message reactions - - [ ] Add file upload support - - [ ] Add image display - - [ ] Add link previews - - [ ] Add emoji support - - [ ] Add message search - - [ ] Add conversation export - -- [ ] **Avatar Enhancements**: - - [ ] Add multiple avatar options - - [ ] Add avatar customization - - [ ] Add background options - - [ ] Add lighting controls - - [ ] Add camera angle options - -- [ ] **Accessibility Enhancements**: - - [ ] Add screen reader announcements - - [ ] Add high contrast mode - - [ ] Add font size controls - - [ ] Add keyboard shortcuts - - [ ] Add voice commands - -### 11. Admin & Management - -- [ ] **Tenant Admin Console**: - - [ ] Create admin UI - - [ ] Add tenant management - - [ ] Add user management - - [ ] Add configuration management - - [ ] Add analytics dashboard - - [ ] Add usage reports - -- [ ] **Content Management**: - - [ ] Add knowledge base management UI - - [ ] Add document upload interface - - [ ] Add content moderation tools - - [ ] Add FAQ management - - [ ] Add prompt template editor - -- [ ] **Monitoring Dashboard**: - - [ ] Create operations dashboard - - [ ] Add real-time metrics - - [ ] Add conversation replay - - [ ] Add error tracking - - [ ] Add performance monitoring - -### 12. Compliance & Governance - -- [ ] **Data Retention**: - - [ ] Implement retention policies - - [ ] Add data deletion workflows - - [ ] Add data export functionality - - [ ] Implement GDPR compliance - - [ ] Add CCPA compliance - -- [ ] **Audit Trails**: - - [ ] Enhance audit logging - - [ ] Add audit log viewer - - [ ] Implement audit log retention - - [ ] Add compliance reports - - [ ] Add tamper detection - -- [ ] **Consent Management**: - - [ ] Add consent tracking - - [ ] Implement consent workflows - - [ ] Add consent withdrawal - - [ ] Add consent reporting - ---- - -## Low Priority Tasks - -### 13. Advanced Features - -- [ ] **Proactive Engagement**: - - [ ] Add proactive notifications - - [ ] Implement scheduled conversations - - [ ] Add event-triggered engagement - - [ ] Add personalized recommendations - -- [ ] **Human Handoff**: - - [ ] Implement handoff workflow - - [ ] Add live agent integration - - [ ] Add handoff queue management - - [ ] Add seamless transition - -- [ ] **Analytics & Insights**: - - [ ] Add conversation analytics - - [ ] Add sentiment analysis - - [ ] Add intent tracking - - [ ] Add satisfaction scoring - - [ ] Add predictive analytics - -- [ ] **Integration Enhancements**: - - [ ] Add webhook support - - [ ] Add API webhooks - - [ ] Add third-party integrations - - [ ] Add CRM integration - - [ ] Add ticketing system integration - -### 14. Developer Experience - -- [ ] **SDK Development**: - - [ ] Create JavaScript SDK - - [ ] Create Python SDK - - [ ] Add SDK documentation - - [ ] Add SDK examples - -- [ ] **API Documentation**: - - [ ] Add OpenAPI/Swagger spec - - [ ] Add interactive API docs - - [ ] Add code examples - - [ ] Add integration guides - -- [ ] **Development Tools**: - - [ ] Add local development setup - - [ ] Add mock services for testing - - [ ] Add development scripts - - [ ] Add debugging tools - ---- - -## Recommendations - -### Architecture Recommendations - -1. **Service Mesh**: Consider implementing a service mesh (Istio, Linkerd) for: - - Service discovery - - Load balancing - - Circuit breaking - - Observability - -2. **Message Queue**: Consider adding a message queue (Kafka, RabbitMQ) for: - - Async processing - - Event streaming - - Decoupling services - - Scalability - -3. **API Gateway**: Consider adding an API gateway (Kong, AWS API Gateway) for: - - Rate limiting - - Authentication - - Request routing - - API versioning - -4. **CDN**: Use a CDN for widget assets: - - Faster load times - - Global distribution - - Reduced server load - - Better caching - -### Performance Recommendations - -1. **Database**: - - Use read replicas for queries - - Implement connection pooling - - Add query result caching - - Consider TimescaleDB for time-series data - -2. **Caching**: - - Cache tenant configurations - - Cache RAG embeddings - - Cache frequently accessed data - - Use Redis Cluster for high availability - -3. **Scaling**: - - Implement horizontal scaling - - Use auto-scaling based on metrics - - Separate GPU cluster for avatars - - Use load balancers - -### Security Recommendations - -1. **Network Security**: - - Use private networks for internal communication - - Implement network segmentation - - Use VPN for admin access - - Add DDoS protection - -2. **Application Security**: - - Regular security audits - - Penetration testing - - Dependency scanning - - Code review process - -3. **Data Security**: - - Encrypt data at rest - - Encrypt data in transit - - Implement key rotation - - Add data masking for non-production - -### Cost Optimization Recommendations - -1. **Resource Management**: - - Right-size instances - - Use spot instances for non-critical workloads - - Implement resource quotas - - Monitor and optimize costs - -2. **API Costs**: - - Cache LLM responses where appropriate - - Optimize ASR/TTS usage - - Use cheaper models for simple queries - - Implement usage limits - -3. **Avatar Rendering**: - - Use GPU instance pooling - - Implement instance reuse - - Optimize rendering settings - - Consider client-side rendering for some use cases - ---- - -## Suggestions for Enhancement - -### User Experience - -1. **Personalization**: - - Learn user preferences - - Adapt conversation style - - Remember past interactions - - Provide personalized recommendations - -2. **Multi-modal Interaction**: - - Add screen sharing - - Add document co-browsing - - Add form filling assistance - - Add visual aids - -3. **Gamification**: - - Add achievement system - - Add progress tracking - - Add rewards for engagement - - Add leaderboards - -### Business Features - -1. **Analytics Dashboard**: - - Real-time metrics - - Historical trends - - User behavior analysis - - ROI calculations - -2. **A/B Testing**: - - Test different prompts - - Test different avatars - - Test different conversation flows - - Test different tool configurations - -3. **White-label Solution**: - - Custom branding - - Custom domain - - Custom styling - - Custom features - -### Technical Enhancements - -1. **Edge Computing**: - - Deploy closer to users - - Reduce latency - - Improve performance - - Better user experience - -2. **Federated Learning**: - - Improve models without sharing data - - Privacy-preserving ML - - Better personalization - - Reduced data transfer - -3. **Blockchain Integration**: - - Immutable audit logs - - Decentralized identity - - Smart contracts for payments - - Trust verification - ---- - -## Testing Tasks - -### Unit Testing -- [ ] Session service (100% coverage) -- [ ] Orchestrator (all state transitions) -- [ ] LLM gateway (all providers) -- [ ] RAG service (retrieval, ranking) -- [ ] Tool executor (all tools) -- [ ] Banking tools (all operations) -- [ ] Safety filters (all rules) -- [ ] Rate limiter (all scenarios) - -### Integration Testing -- [ ] API endpoints (all routes) -- [ ] WebSocket connections -- [ ] Database operations -- [ ] Redis operations -- [ ] Service interactions -- [ ] Error handling -- [ ] Retry logic - -### E2E Testing -- [ ] Widget initialization -- [ ] Session lifecycle -- [ ] Text conversation -- [ ] Voice conversation -- [ ] Tool execution -- [ ] Error scenarios -- [ ] Multi-tenant isolation - -### Performance Testing -- [ ] Load testing (1000+ concurrent sessions) -- [ ] Stress testing -- [ ] Endurance testing -- [ ] Spike testing -- [ ] Volume testing - -### Security Testing -- [ ] Penetration testing -- [ ] Vulnerability scanning -- [ ] Authentication testing -- [ ] Authorization testing -- [ ] Input validation testing -- [ ] SQL injection testing -- [ ] XSS testing - ---- - -## Documentation Tasks - -- [ ] **API Documentation**: - - [ ] Complete OpenAPI specification - - [ ] Add request/response examples - - [ ] Add error code documentation - - [ ] Add authentication guide - -- [ ] **Integration Guides**: - - [ ] Widget integration guide (enhanced) - - [ ] Banking service integration guide - - [ ] Third-party service integration - - [ ] Custom tool development guide - -- [ ] **Operations Documentation**: - - [ ] Deployment runbook - - [ ] Troubleshooting guide - - [ ] Monitoring guide - - [ ] Incident response guide - -- [ ] **Developer Documentation**: - - [ ] Architecture deep dive - - [ ] Code contribution guide - - [ ] Development setup guide - - [ ] Testing guide - ---- - -## Production Readiness Checklist - -### Infrastructure -- [ ] Production database setup -- [ ] Production Redis setup -- [ ] Load balancer configuration -- [ ] CDN configuration -- [ ] DNS configuration -- [ ] SSL/TLS certificates -- [ ] Backup systems -- [ ] Disaster recovery plan - -### Security -- [ ] Security audit completed -- [ ] Penetration testing passed -- [ ] Secrets management configured -- [ ] Access controls implemented -- [ ] Monitoring and alerting active -- [ ] Incident response plan ready - -### Monitoring -- [ ] Metrics collection active -- [ ] Logging configured -- [ ] Tracing enabled -- [ ] Dashboards created -- [ ] Alerts configured -- [ ] On-call rotation set up - -### Performance -- [ ] Load testing completed -- [ ] Performance benchmarks met -- [ ] Scaling configured -- [ ] Caching optimized -- [ ] Database optimized - -### Compliance -- [ ] GDPR compliance verified -- [ ] CCPA compliance verified -- [ ] Data retention policies set -- [ ] Audit logging active -- [ ] Consent management implemented - -### Documentation -- [ ] API documentation complete -- [ ] Integration guides complete -- [ ] Operations runbooks complete -- [ ] Troubleshooting guides complete - ---- - -## Summary Statistics - -- **Total Completed Tasks**: 50+ -- **Critical Tasks Remaining**: 12 -- **High Priority Tasks**: 20+ -- **Medium Priority Tasks**: 15+ -- **Low Priority Tasks**: 10+ -- **Recommendations**: 15+ -- **Suggestions**: 10+ - -**Estimated Time to Production**: 10-16 days (with focused effort) - ---- - -## Priority Order for Next Steps - -1. **Week 1**: Replace mock services (ASR, TTS, LLM) -2. **Week 2**: Complete WebRTC implementation -3. **Week 3**: Unreal Engine avatar setup -4. **Week 4**: Testing and production hardening - ---- - -**Last Updated**: 2025-01-20 -**Status**: Ready for production integration phase - diff --git a/virtual-banker/COMPLETION_SUMMARY.md b/virtual-banker/COMPLETION_SUMMARY.md deleted file mode 100644 index f686e20..0000000 --- a/virtual-banker/COMPLETION_SUMMARY.md +++ /dev/null @@ -1,149 +0,0 @@ -# Virtual Banker Implementation - Completion Summary - -## ✅ All Integration Steps Completed - -### 1. Service Integration ✅ -- **Orchestrator** fully integrated with: - - LLM Gateway (with conversation history) - - RAG Service (document retrieval) - - Tool Executor (banking tools) - - ASR/TTS services - -### 2. Banking Service Integration ✅ -- **BankingClient** created for HTTP communication -- **AccountStatusTool** connects to banking API with fallback -- **CreateTicketTool** connects to banking API with fallback -- All tools have graceful fallback to mock data - -### 3. WebSocket/Realtime Support ✅ -- **Realtime Gateway** integrated into API routes -- WebSocket endpoint: `/v1/realtime/{session_id}` -- Connection management and message routing - -### 4. Startup Scripts ✅ -- `scripts/setup-database.sh` - Database migration runner -- `scripts/start-backend.sh` - Backend service starter -- Both scripts are executable and ready to use - -### 5. Code Quality ✅ -- All compilation errors fixed -- Dependencies properly managed -- Code compiles successfully - -## 🎯 System Status - -### Backend Services -- ✅ Session Management -- ✅ REST API (sessions, health) -- ✅ WebSocket Gateway -- ✅ Conversation Orchestrator -- ✅ LLM Gateway (mock, ready for OpenAI/Anthropic) -- ✅ RAG Service (pgvector) -- ✅ Tool Framework -- ✅ Banking Tool Integrations -- ✅ ASR Service (mock, ready for Deepgram) -- ✅ TTS Service (mock, ready for ElevenLabs) -- ✅ Safety/Compliance -- ✅ Memory Service -- ✅ Observability - -### Frontend Widget -- ✅ React/TypeScript components -- ✅ Chat UI -- ✅ Voice controls -- ✅ Avatar view -- ✅ Captions -- ✅ Settings -- ✅ PostMessage API -- ✅ WebRTC hooks (ready for connection) - -### Infrastructure -- ✅ Database migrations -- ✅ Docker configurations -- ✅ Deployment scripts -- ✅ Documentation - -## 📋 Next Steps (For Production) - -### Immediate (Ready to Implement) -1. **Replace Mock Services**: - - Add API keys for Deepgram/ElevenLabs/OpenAI - - Update service implementations - - Test with real APIs - -2. **Complete WebRTC**: - - Implement SDP offer/answer - - Add ICE candidate handling - - Test media streaming - -3. **Unreal Engine Setup**: - - Create Unreal project - - Import digital human - - Configure PixelStreaming - -### Testing -- Unit tests for services -- Integration tests for API -- E2E tests for widget - -### Production -- Secrets management -- Monitoring setup -- Scaling configuration -- Security hardening - -## 🚀 Quick Start - -```bash -# 1. Setup database -cd virtual-banker -./scripts/setup-database.sh - -# 2. Start backend -./scripts/start-backend.sh - -# 3. Test API -curl http://localhost:8081/health - -# 4. Create session -curl -X POST http://localhost:8081/v1/sessions \ - -H "Content-Type: application/json" \ - -d '{ - "tenant_id": "default", - "user_id": "test-user", - "auth_assertion": "test-token" - }' -``` - -## 📊 Implementation Statistics - -- **Backend Files**: 30+ Go files -- **Frontend Files**: 20+ TypeScript/React files -- **Database Migrations**: 5 migration files -- **Documentation**: 4 comprehensive guides -- **Scripts**: 2 startup scripts -- **Total Lines of Code**: ~5000+ - -## ✨ Key Features Implemented - -1. **Multi-tenant Architecture** - Complete tenant isolation -2. **Session Management** - Secure, ephemeral sessions -3. **Real-time Communication** - WebSocket infrastructure -4. **Conversation Orchestration** - State machine with barge-in -5. **RAG Integration** - Vector search for knowledge retrieval -6. **Tool Framework** - Extensible action system -7. **Banking Integration** - Connected to backend services -8. **Safety & Compliance** - Content filtering, rate limiting -9. **Observability** - Tracing and metrics -10. **Accessibility** - WCAG-compliant widget - -## 🎉 Status: READY FOR INTEGRATION - -All core infrastructure is complete and functional. The system is ready for: -- Integration with real ASR/TTS/LLM services -- Connection to production banking APIs -- Unreal Engine avatar setup -- Production deployment - -The Virtual Banker submodule is **fully implemented** and ready for the next phase of development! - diff --git a/virtual-banker/FINAL_STATUS.md b/virtual-banker/FINAL_STATUS.md deleted file mode 100644 index 1769004..0000000 --- a/virtual-banker/FINAL_STATUS.md +++ /dev/null @@ -1,188 +0,0 @@ -# Virtual Banker - Final Implementation Status - -## ✅ ALL INTEGRATION STEPS COMPLETE - -**Date**: 2025-01-20 -**Status**: ✅ **FULLY INTEGRATED AND BUILDING SUCCESSFULLY** - ---- - -## Completed Integration Tasks - -### 1. Service Integration ✅ -- **Orchestrator** fully connected to: - - LLM Gateway (with conversation history tracking) - - RAG Service (document retrieval with pgvector) - - Tool Executor (banking tool execution) - - ASR/TTS services (ready for real API integration) - -### 2. Banking Service Integration ✅ -- **BankingClient** HTTP client created -- **AccountStatusTool** connects to `backend/banking/accounts/` API -- **CreateTicketTool** connects to banking ticket API -- All tools have graceful fallback to mock data if service unavailable -- Integration points ready for production banking endpoints - -### 3. WebSocket/Realtime Support ✅ -- **Realtime Gateway** integrated into API server -- WebSocket endpoint: `GET /v1/realtime/{session_id}` -- Connection management and message routing implemented -- Ready for full WebRTC signaling implementation - -### 4. Startup Scripts ✅ -- `scripts/setup-database.sh` - Runs all database migrations -- `scripts/start-backend.sh` - Starts backend with proper environment -- Both scripts are executable and tested - -### 5. Code Quality ✅ -- All compilation errors fixed -- All imports properly managed -- Code builds successfully: `✅ Build successful!` -- No linting errors - ---- - -## System Architecture (Fully Integrated) - -``` -┌─────────────────────────────────────────┐ -│ Embeddable Widget │ -│ (React/TypeScript) │ -└──────────────┬──────────────────────────┘ - │ HTTP/WebSocket - ▼ -┌─────────────────────────────────────────┐ -│ API Server │ -│ - Session Management │ -│ - REST Endpoints │ -│ - WebSocket Gateway │ -└──────────────┬──────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────┐ -│ Conversation Orchestrator │ -│ ┌──────────┐ ┌──────────┐ │ -│ │ LLM │ │ RAG │ │ -│ │ Gateway │ │ Service │ │ -│ └──────────┘ └──────────┘ │ -│ ┌──────────┐ ┌──────────┐ │ -│ │ Tools │ │ ASR/TTS │ │ -│ │Executor │ │ Services │ │ -│ └──────────┘ └──────────┘ │ -└──────────────┬──────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────┐ -│ Banking Services │ -│ (backend/banking/) │ -└─────────────────────────────────────────┘ -``` - ---- - -## Build Status - -```bash -✅ Backend compiles successfully -✅ All dependencies resolved -✅ No compilation errors -✅ Ready for deployment -``` - ---- - -## Quick Start - -```bash -# 1. Setup database -cd virtual-banker -./scripts/setup-database.sh - -# 2. Start backend -./scripts/start-backend.sh - -# 3. Test health endpoint -curl http://localhost:8081/health - -# 4. Create a session -curl -X POST http://localhost:8081/v1/sessions \ - -H "Content-Type: application/json" \ - -d '{ - "tenant_id": "default", - "user_id": "test-user", - "auth_assertion": "test-token" - }' -``` - ---- - -## Integration Points - -### Banking Services -- **Account Status**: `GET /api/v1/banking/accounts/{id}` -- **Create Ticket**: `POST /api/v1/banking/tickets` -- **Fallback**: Mock data if service unavailable - -### External APIs (Ready for Integration) -- **ASR**: Deepgram/Google STT (mock → real) -- **TTS**: ElevenLabs/Azure TTS (mock → real) -- **LLM**: OpenAI/Anthropic (mock → real) - -### WebSocket -- **Endpoint**: `ws://localhost:8081/v1/realtime/{session_id}` -- **Purpose**: Real-time signaling for WebRTC -- **Status**: Infrastructure ready, signaling implementation pending - ---- - -## File Statistics - -- **Backend Go Files**: 30+ -- **Frontend React/TS Files**: 20+ -- **Database Migrations**: 5 -- **Documentation Files**: 6 -- **Scripts**: 2 -- **Total Lines**: ~5000+ - ---- - -## Next Steps for Production - -1. **Replace Mock Services** (1-2 days) - - Add API keys - - Update service implementations - - Test with real APIs - -2. **Complete WebRTC** (2-3 days) - - Implement SDP offer/answer - - Add ICE candidate handling - - Test media streaming - -3. **Unreal Engine Setup** (3-5 days) - - Create project - - Import character - - Configure PixelStreaming - -4. **Testing** (2-3 days) - - Unit tests - - Integration tests - - E2E tests - -5. **Production Deployment** (2-3 days) - - Secrets management - - Monitoring - - Scaling - -**Total Estimated Time to Production**: 10-16 days - ---- - -## ✅ Status: READY FOR PRODUCTION INTEGRATION - -All core infrastructure is complete, integrated, and building successfully. The system is ready for: -- Real API integrations -- Production deployment -- Further development - -**The Virtual Banker submodule is fully implemented and operational!** 🎉 - diff --git a/virtual-banker/IMPLEMENTATION_SUMMARY.md b/virtual-banker/IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index cf646e2..0000000 --- a/virtual-banker/IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,162 +0,0 @@ -# Virtual Banker Implementation Summary - -## Status: ✅ COMPLETE - -All phases of the Virtual Banker submodule have been implemented according to the plan. - -## What Was Built - -### Phase 0: Foundation & Widget ✅ -- **Backend Structure**: Complete Go backend with session management -- **REST API**: Full API with session creation, token refresh, session ending -- **Database Migrations**: All tables for sessions, tenants, conversations, knowledge base, user profiles -- **Embeddable Widget**: Complete React/TypeScript widget with: - - Chat UI with accessibility features - - Voice controls (push-to-talk, hands-free) - - Avatar view component - - Captions support - - Settings panel - - PostMessage API for host integration -- **Integration**: Added to main docker-compose.yml - -### Phase 1: Voice & Realtime ✅ -- **WebRTC Gateway**: WebSocket-based signaling infrastructure -- **ASR Service**: Interface and mock implementation (ready for Deepgram/Google STT integration) -- **TTS Service**: Interface and mock implementation (ready for ElevenLabs/Azure TTS integration) -- **Orchestrator**: Complete conversation orchestrator with: - - State machine (IDLE → LISTENING → THINKING → SPEAKING) - - Barge-in support (interrupt handling) - - Audio/video synchronization - -### Phase 2: LLM & RAG ✅ -- **LLM Gateway**: Interface and mock (ready for OpenAI/Anthropic integration) -- **Prompt Builder**: Multi-tenant prompt assembly with RAG context injection -- **RAG Service**: Complete implementation with pgvector: - - Document ingestion - - Vector similarity search - - Citation formatting -- **Tool Framework**: Complete tool system with: - - Tool registry - - Executor with audit logging - - Banking tool integrations: - - get_account_status - - create_support_ticket - - schedule_appointment - - submit_payment - -### Phase 3: Avatar System ✅ -- **Unreal Engine Setup**: Complete documentation and structure -- **Renderer Service**: PixelStreaming integration service -- **Animation Controller**: Complete animation system: - - Viseme mapping (phoneme → viseme) - - Expression system (valence/arousal → facial expressions) - - Gesture system (rule-based gesture selection) - -### Phase 4: Memory & Observability ✅ -- **Memory Service**: User profiles and conversation history -- **Observability**: Tracing and metrics collection -- **Safety/Compliance**: Content filtering and rate limiting - -### Phase 5: Enterprise Features ✅ -- **Multi-tenancy**: Complete tenant configuration system -- **Compliance**: Safety filters, PII redaction, rate limiting -- **Documentation**: Complete docs for architecture, API, widget integration, deployment - -## File Structure - -``` -virtual-banker/ -├── backend/ -│ ├── session/ ✅ Session management -│ ├── orchestrator/ ✅ Conversation orchestration -│ ├── llm/ ✅ LLM gateway & prompts -│ ├── rag/ ✅ RAG service -│ ├── tools/ ✅ Tool framework + banking tools -│ ├── asr/ ✅ Speech-to-text service -│ ├── tts/ ✅ Text-to-speech service -│ ├── safety/ ✅ Content filtering & rate limiting -│ ├── memory/ ✅ User profiles & history -│ ├── observability/ ✅ Tracing & metrics -│ ├── api/ ✅ REST API routes -│ ├── realtime/ ✅ WebRTC gateway -│ └── main.go ✅ Entry point -├── widget/ -│ ├── src/ -│ │ ├── components/ ✅ All UI components -│ │ ├── hooks/ ✅ React hooks -│ │ ├── services/ ✅ API & WebRTC clients -│ │ └── types/ ✅ TypeScript types -│ └── public/ ✅ Loader script -├── avatar/ -│ ├── renderer/ ✅ Render service -│ ├── animation/ ✅ Animation controllers -│ └── unreal/ ✅ Unreal setup docs -├── database/ -│ └── migrations/ ✅ All migrations -├── deployment/ ✅ Docker configs -└── docs/ ✅ Complete documentation -``` - -## Next Steps for Production - -1. **Integrate Real Services**: - - Replace ASR mock with Deepgram/Google STT - - Replace TTS mock with ElevenLabs/Azure TTS - - Replace LLM mock with OpenAI/Anthropic - - Connect banking tools to actual backend/banking/ services - -2. **Complete WebRTC**: - - Implement full WebRTC signaling - - Add TURN server configuration - - Complete media streaming - -3. **Unreal Engine Setup**: - - Create actual Unreal project - - Import digital human character - - Configure PixelStreaming - - Package for deployment - -4. **Testing**: - - Unit tests for all services - - Integration tests - - E2E tests for widget - -5. **Production Hardening**: - - Secrets management - - Monitoring & alerting - - Scaling configuration - - Security audit - -## Key Features Implemented - -✅ Multi-tenant architecture -✅ JWT/SSO authentication -✅ Real-time voice interaction (infrastructure) -✅ Full video realism (Unreal integration ready) -✅ Embeddable widget -✅ Accessibility (WCAG-oriented) -✅ Safety & compliance -✅ Observability -✅ Tool framework with banking integrations -✅ RAG for knowledge retrieval -✅ User memory & profiles - -## Documentation - -All documentation is complete: -- ✅ Architecture overview -- ✅ API reference -- ✅ Widget integration guide -- ✅ Deployment guide -- ✅ Unreal Engine setup guide - -## Ready for Integration - -The Virtual Banker submodule is ready for: -1. Integration with real ASR/TTS/LLM services -2. Connection to existing banking services -3. Unreal Engine avatar setup -4. Production deployment - -All core infrastructure is in place and functional with mock implementations that can be swapped for real services. - diff --git a/virtual-banker/NEXT_STEPS.md b/virtual-banker/NEXT_STEPS.md deleted file mode 100644 index 5b85e49..0000000 --- a/virtual-banker/NEXT_STEPS.md +++ /dev/null @@ -1,162 +0,0 @@ -# Next Steps for Virtual Banker - -## ✅ Completed Integration Steps - -1. **Service Integration**: All services are now connected: - - Orchestrator → LLM Gateway - - Orchestrator → RAG Service - - Orchestrator → Tool Executor - - Banking tools → Banking API client - -2. **WebSocket Support**: Realtime gateway integrated into API routes - -3. **Startup Scripts**: Created scripts for database setup and backend startup - -4. **Banking Integration**: Tools now connect to backend banking services with fallback - -## 🔄 Remaining Integration Tasks - -### 1. Replace Mock Services with Real APIs - -**ASR Service** (Speech-to-Text): -- [ ] Integrate Deepgram API - - Get API key from Deepgram - - Update `backend/asr/service.go` to use Deepgram streaming API - - Test with real audio streams - -- [ ] Or integrate Google Speech-to-Text - - Set up Google Cloud credentials - - Implement streaming transcription - -**TTS Service** (Text-to-Speech): -- [ ] Integrate ElevenLabs API - - Get API key from ElevenLabs - - Update `backend/tts/service.go` to use ElevenLabs API - - Configure voice selection per tenant - -- [ ] Or integrate Azure TTS - - Set up Azure credentials - - Implement SSML support - -**LLM Gateway**: -- [ ] Integrate OpenAI API - - Get API key - - Update `backend/llm/gateway.go` to use OpenAI - - Implement function calling - - Add streaming support - -- [ ] Or integrate Anthropic Claude - - Get API key - - Implement tool use - -### 2. Complete WebRTC Implementation - -- [ ] Implement full WebRTC signaling - - SDP offer/answer exchange - - ICE candidate handling - - TURN server configuration - -- [ ] Add media streaming - - Audio stream from client → ASR - - Audio stream from TTS → client - - Video stream from avatar → client - -### 3. Connect to Existing Banking Services - -Update banking tool integrations to match actual API endpoints: - -```go -// Check actual endpoints in backend/banking/ -// Update integration.go with correct paths -``` - -### 4. Unreal Engine Avatar Setup - -- [ ] Install Unreal Engine 5.3+ -- [ ] Create new project -- [ ] Enable PixelStreaming plugin -- [ ] Import digital human character -- [ ] Set up blendshapes for visemes -- [ ] Configure animation blueprints -- [ ] Package for Linux deployment - -### 5. Testing - -- [ ] Unit tests for all services -- [ ] Integration tests for API endpoints -- [ ] E2E tests for widget -- [ ] Load testing for concurrent sessions - -### 6. Production Deployment - -- [ ] Set up secrets management -- [ ] Configure monitoring (Prometheus/Grafana) -- [ ] Set up logging aggregation -- [ ] Configure auto-scaling -- [ ] Security audit -- [ ] Performance optimization - -## Quick Start Commands - -```bash -# Setup database -cd virtual-banker -./scripts/setup-database.sh - -# Start backend -./scripts/start-backend.sh - -# Build widget -cd widget -npm install -npm run build -``` - -## Environment Variables - -Create `.env` file: - -```bash -DATABASE_URL=postgres://explorer:changeme@localhost:5432/explorer?sslmode=disable -REDIS_URL=redis://localhost:6379 -PORT=8081 - -# For real services (when ready): -DEEPGRAM_API_KEY=your_key_here -ELEVENLABS_API_KEY=your_key_here -OPENAI_API_KEY=your_key_here -BANKING_API_URL=http://localhost:8080 -``` - -## Testing the Integration - -1. **Start services**: - ```bash - docker-compose up -d postgres redis - ./scripts/start-backend.sh - ``` - -2. **Create a session**: - ```bash - curl -X POST http://localhost:8081/v1/sessions \ - -H "Content-Type: application/json" \ - -d '{ - "tenant_id": "default", - "user_id": "test-user", - "auth_assertion": "test-token" - }' - ``` - -3. **Test WebSocket**: - ```bash - # Use wscat or similar tool - wscat -c ws://localhost:8081/v1/realtime/{session_id} - ``` - -## Notes - -- All mock services are functional and can be tested independently -- Banking tools have fallback to mock data if service unavailable -- WebRTC gateway is ready but needs full signaling implementation -- Widget is fully functional for text chat (voice requires WebRTC completion) - diff --git a/virtual-banker/README.md b/virtual-banker/README.md deleted file mode 100644 index f0761ea..0000000 --- a/virtual-banker/README.md +++ /dev/null @@ -1,190 +0,0 @@ -# Virtual Banker Submodule - -AI digital twin human-like Virtual Banker with full video realism using Unreal Engine server-rendered avatars, real-time voice interaction, and an embeddable widget for portal sites. - -## Features - -- **Embeddable Widget**: Drop-in widget for any portal site -- **Real-time Voice**: ASR (Speech-to-Text) and TTS (Text-to-Speech) with streaming -- **Full Video Realism**: Unreal Engine server-rendered avatar with PixelStreaming -- **Multi-tenant**: Different configs/brands/policies per tenant -- **Secure Auth**: JWT/SSO integration -- **Accessible**: WCAG-oriented (keyboard, screen reader, captions, reduced motion) -- **Observable**: Audit logs, safety rules, analytics - -## Architecture - -``` -virtual-banker/ -├── backend/ # Go backend services -├── widget/ # React/TypeScript embeddable widget -├── avatar/ # Unreal Engine avatar service -├── database/ # Database migrations -└── deployment/ # Docker/Kubernetes configs -``` - -## Quick Start - -### Prerequisites - -- Docker and Docker Compose -- Go 1.21+ (for backend development) -- Node.js 20+ (for widget development) -- PostgreSQL 16+ with pgvector extension -- Redis - -### Development Setup - -1. **Start infrastructure** (uses existing postgres/redis from main monorepo): -```bash -cd deployment -docker-compose up -d postgres redis -``` - -2. **Run database migrations**: -```bash -cd database -psql -U explorer -d explorer -f migrations/001_sessions.up.sql -psql -U explorer -d explorer -f migrations/002_conversations.up.sql -psql -U explorer -d explorer -f migrations/003_tenants.up.sql -psql -U explorer -d explorer -f migrations/004_vector_extension.up.sql -psql -U explorer -d explorer -f migrations/005_user_profiles.up.sql -``` - -3. **Start backend API**: -```bash -cd backend -go run main.go -``` - -4. **Build and serve widget**: -```bash -cd widget -npm install -npm run build -# Serve dist/ via CDN or static server -``` - -## Widget Integration - -### Basic Integration - -Add the widget loader script to your HTML: - -```html - -
-``` - -### Programmatic Control - -```javascript -// Open widget -window.VirtualBankerWidgetAPI.open(); - -// Close widget -window.VirtualBankerWidgetAPI.close(); - -// Set context -window.VirtualBankerWidgetAPI.setContext({ - route: '/account', - accountId: 'acc-123' -}); - -// Update auth token -window.VirtualBankerWidgetAPI.setAuthToken('new-jwt-token'); -``` - -## API Endpoints - -### Create Session -``` -POST /v1/sessions -{ - "tenant_id": "tenant-123", - "user_id": "user-456", - "auth_assertion": "jwt-token" -} -``` - -### Refresh Token -``` -POST /v1/sessions/{id}/refresh-token -``` - -### End Session -``` -POST /v1/sessions/{id}/end -``` - -## Implementation Status - -### Phase 0: Foundation & Widget ✅ -- [x] Backend session service -- [x] REST API endpoints -- [x] Database migrations -- [x] Embeddable widget (React/TypeScript) -- [x] Basic chat UI -- [x] Theming system -- [x] Accessibility features - -### Phase 1: Voice & Realtime ✅ -- [x] WebRTC infrastructure -- [x] ASR service integration (mock + interface for Deepgram) -- [x] TTS service integration (mock + interface for ElevenLabs) -- [x] Conversation orchestrator -- [x] Barge-in support - -### Phase 2: LLM & RAG ✅ -- [x] LLM gateway (mock + interface for OpenAI) -- [x] RAG service with pgvector -- [x] Tool framework -- [x] Banking integrations - -### Phase 3: Avatar System ✅ -- [x] Unreal Engine setup documentation -- [x] Render service structure -- [x] Animation controller (visemes, expressions, gestures) - -### Phase 4: Memory & Observability ✅ -- [x] Memory service -- [x] Observability (tracing, metrics) -- [x] Safety/compliance filters - -### Phase 5: Enterprise Features (In Progress) -- [x] Multi-tenancy support -- [ ] Tenant admin console (UI) -- [ ] Advanced compliance tools -- [ ] Usage analytics dashboard - -## Next Steps - -1. **Integrate Real Services**: Replace mocks with actual API integrations: - - Deepgram or Google STT for ASR - - ElevenLabs or Azure TTS for TTS - - OpenAI or Anthropic for LLM - - Connect to existing banking services - -2. **Complete WebRTC**: Implement full WebRTC signaling and media streaming - -3. **Unreal Setup**: Set up actual Unreal Engine project with digital human - -4. **Testing**: Add unit tests, integration tests, E2E tests - -5. **Production Deployment**: Configure for production with proper secrets, monitoring, scaling - -## Documentation - -- [Architecture](./docs/ARCHITECTURE.md) -- [API Reference](./docs/API.md) -- [Widget Integration](./docs/WIDGET_INTEGRATION.md) -- [Deployment](./docs/DEPLOYMENT.md) - -## License - -MIT diff --git a/virtual-banker/TASK_SUMMARY.md b/virtual-banker/TASK_SUMMARY.md deleted file mode 100644 index e95106f..0000000 --- a/virtual-banker/TASK_SUMMARY.md +++ /dev/null @@ -1,121 +0,0 @@ -# Virtual Banker - Task Summary (Quick Reference) - -## ✅ Completed: 50+ Tasks - -**All core implementation complete** - System is functional with mock services - -## 🔴 Critical (Must Do Before Production): 12 Tasks - -1. **Replace ASR Mock** → Deepgram or Google STT -2. **Replace TTS Mock** → ElevenLabs or Azure TTS -3. **Replace LLM Mock** → OpenAI or Anthropic -4. **Complete WebRTC Signaling** → SDP offer/answer, ICE candidates -5. **Set Up TURN Server** → For NAT traversal -6. **Implement Media Streaming** → Audio/video streams -7. **Unreal Engine Setup** → Create project, import character, configure PixelStreaming -8. **Package Unreal Project** → For Linux deployment -9. **Connect Banking APIs** → Update endpoints to match actual services -10. **Security Audit** → Penetration testing, vulnerability scanning -11. **Secrets Management** → Vault or AWS Secrets Manager -12. **Production Monitoring** → Prometheus, Grafana, alerting - -## 🟠 High Priority: 20+ Tasks - -### Testing (8 tasks) -- Unit tests for all services -- Integration tests -- E2E tests -- Load testing -- Security testing - -### Security (6 tasks) -- JWT validation enhancement -- Input validation -- PII detection/redaction -- Content filtering enhancement -- Network security -- Application security audit - -### Monitoring (6 tasks) -- Prometheus metrics -- Grafana dashboards -- Centralized logging -- Distributed tracing -- Alerting rules -- Performance monitoring - -## 🟡 Medium Priority: 15+ Tasks - -- Multi-language support -- Advanced RAG (reranking, hybrid search) -- Enhanced tool framework -- Conversation features -- Widget enhancements -- Avatar enhancements -- Tenant admin console -- Content management UI -- Compliance features - -## 🟢 Low Priority: 10+ Tasks - -- Proactive engagement -- Human handoff -- Analytics & insights -- SDK development -- API documentation -- Development tools - -## 📊 Statistics - -- **Files Created**: 59 total files -- **Code Files**: 40 (Go, TypeScript, React) -- **Lines of Code**: ~5,000+ -- **Documentation**: 6 comprehensive guides -- **Migrations**: 5 database migrations -- **Scripts**: 2 startup scripts - -## ⏱️ Time Estimates - -- **Critical Tasks**: 10-16 days -- **High Priority**: 2-3 weeks -- **Medium Priority**: 1-2 months -- **Low Priority**: Ongoing - -## 🎯 Recommended Next Steps (Priority Order) - -### Week 1: Real Service Integration -1. Get API keys (Deepgram, ElevenLabs, OpenAI) -2. Replace ASR mock -3. Replace TTS mock -4. Replace LLM mock -5. Test with real APIs - -### Week 2: WebRTC Completion -1. Implement SDP signaling -2. Add ICE candidate handling -3. Set up TURN server -4. Test media streaming - -### Week 3: Avatar Setup -1. Install Unreal Engine -2. Create project -3. Import character -4. Configure PixelStreaming -5. Package for deployment - -### Week 4: Production Hardening -1. Security audit -2. Testing suite -3. Monitoring setup -4. Documentation -5. Deployment - -## 📋 Full Details - -See `COMPLETE_TASK_LIST.md` for: -- Detailed task descriptions -- Recommendations -- Suggestions for enhancement -- Testing requirements -- Production readiness checklist - diff --git a/virtual-banker/avatar/Dockerfile b/virtual-banker/avatar/Dockerfile deleted file mode 100644 index a5c942e..0000000 --- a/virtual-banker/avatar/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Dockerfile for Unreal Engine Avatar Renderer -# Note: This is a placeholder - actual Unreal deployment requires: -# 1. Packaged Unreal project -# 2. NVIDIA GPU support -# 3. CUDA drivers -# 4. Custom base image with Unreal runtime - -FROM nvidia/cuda:12.0.0-base-ubuntu22.04 - -# Install dependencies -RUN apt-get update && apt-get install -y \ - libgl1-mesa-glx \ - libglib2.0-0 \ - libx11-6 \ - libxext6 \ - libxrender1 \ - && rm -rf /var/lib/apt/lists/* - -# Copy Unreal packaged project -# COPY unreal-package/ /app/unreal/ - -# Copy renderer service -COPY renderer/ /app/renderer/ - -WORKDIR /app - -# Expose PixelStreaming port -EXPOSE 8888 - -# Start renderer service (which manages Unreal instances) -CMD ["./renderer/service"] - diff --git a/virtual-banker/avatar/animation/expressions.go b/virtual-banker/avatar/animation/expressions.go deleted file mode 100644 index 2aac2da..0000000 --- a/virtual-banker/avatar/animation/expressions.go +++ /dev/null @@ -1,68 +0,0 @@ -package animation - -// ExpressionMapping maps emotion values to facial expressions -type ExpressionMapping struct { - Valence float64 // -1.0 to 1.0 - Arousal float64 // 0.0 to 1.0 -} - -// GetExpressionFromEmotion maps emotion to expression parameters -func GetExpressionFromEmotion(valence, arousal float64) ExpressionParams { - // Map valence/arousal to expression - // High valence + high arousal = happy/excited - // Low valence + high arousal = angry/frustrated - // High valence + low arousal = calm/content - // Low valence + low arousal = sad/depressed - - var emotion string - var smileAmount float64 - var browRaise float64 - var eyeWideness float64 - - if valence > 0.5 && arousal > 0.5 { - emotion = "happy" - smileAmount = 0.8 - browRaise = 0.3 - eyeWideness = 0.6 - } else if valence < -0.5 && arousal > 0.5 { - emotion = "angry" - smileAmount = -0.5 - browRaise = -0.7 - eyeWideness = 0.8 - } else if valence > 0.3 && arousal < 0.3 { - emotion = "calm" - smileAmount = 0.3 - browRaise = 0.0 - eyeWideness = 0.4 - } else if valence < -0.3 && arousal < 0.3 { - emotion = "sad" - smileAmount = -0.3 - browRaise = 0.2 - eyeWideness = 0.3 - } else { - emotion = "neutral" - smileAmount = 0.0 - browRaise = 0.0 - eyeWideness = 0.5 - } - - return ExpressionParams{ - Emotion: emotion, - SmileAmount: smileAmount, - BrowRaise: browRaise, - EyeWideness: eyeWideness, - Valence: valence, - Arousal: arousal, - } -} - -// ExpressionParams contains facial expression parameters -type ExpressionParams struct { - Emotion string - SmileAmount float64 // -1.0 to 1.0 - BrowRaise float64 // -1.0 to 1.0 - EyeWideness float64 // 0.0 to 1.0 - Valence float64 - Arousal float64 -} - diff --git a/virtual-banker/avatar/animation/gestures.go b/virtual-banker/avatar/animation/gestures.go deleted file mode 100644 index f156b46..0000000 --- a/virtual-banker/avatar/animation/gestures.go +++ /dev/null @@ -1,103 +0,0 @@ -package animation - -// GestureType represents a gesture type -type GestureType string - -const ( - GestureNod GestureType = "nod" - GestureShake GestureType = "shake" - GesturePoint GestureType = "point" - GestureWave GestureType = "wave" - GestureIdle GestureType = "idle" -) - -// GetGestureFromText determines appropriate gesture from text context -func GetGestureFromText(text string, emotion string) []GestureEvent { - var gestures []GestureEvent - - // Simple rule-based gesture selection - // In production, this could use NLP to detect intent - - // Greetings - if containsAny(text, []string{"hello", "hi", "hey", "greetings"}) { - gestures = append(gestures, GestureEvent{ - Type: string(GestureWave), - StartTime: 0.0, - Duration: 1.0, - Intensity: 0.7, - }) - } - - // Affirmations - if containsAny(text, []string{"yes", "correct", "right", "exactly", "sure"}) { - gestures = append(gestures, GestureEvent{ - Type: string(GestureNod), - StartTime: 0.0, - Duration: 0.5, - Intensity: 0.8, - }) - } - - // Negations - if containsAny(text, []string{"no", "not", "wrong", "incorrect"}) { - gestures = append(gestures, GestureEvent{ - Type: string(GestureShake), - StartTime: 0.0, - Duration: 0.5, - Intensity: 0.8, - }) - } - - // Directions/pointing - if containsAny(text, []string{"here", "there", "this", "that", "look"}) { - gestures = append(gestures, GestureEvent{ - Type: string(GesturePoint), - StartTime: 0.2, - Duration: 0.8, - Intensity: 0.6, - }) - } - - // If no specific gesture, add idle - if len(gestures) == 0 { - gestures = append(gestures, GestureEvent{ - Type: string(GestureIdle), - StartTime: 0.0, - Duration: 2.0, - Intensity: 0.3, - }) - } - - return gestures -} - -// GestureEvent represents a gesture event -type GestureEvent struct { - Type string - StartTime float64 - Duration float64 - Intensity float64 -} - -// containsAny checks if text contains any of the given strings -func containsAny(text string, keywords []string) bool { - lowerText := toLower(text) - for _, keyword := range keywords { - if contains(lowerText, toLower(keyword)) { - return true - } - } - return false -} - -// Helper functions (simplified - in production use proper string functions) -func toLower(s string) string { - // Simplified - use strings.ToLower in production - return s -} - -func contains(s, substr string) bool { - // Simplified - use strings.Contains in production - return len(s) >= len(substr) -} - diff --git a/virtual-banker/avatar/animation/visemes.go b/virtual-banker/avatar/animation/visemes.go deleted file mode 100644 index ee4732b..0000000 --- a/virtual-banker/avatar/animation/visemes.go +++ /dev/null @@ -1,113 +0,0 @@ -package animation - -// VisemeMapping maps phonemes to visemes -var VisemeMapping = map[string]string{ - // Silence - "sil": "sil", - "sp": "sil", - - // Vowels - "aa": "aa", // "father" - "ae": "aa", // "cat" - "ah": "aa", // "but" - "ao": "oh", // "law" - "aw": "ou", // "cow" - "ay": "aa", // "hide" - "eh": "ee", // "red" - "er": "er", // "her" - "ey": "ee", // "ate" - "ih": "ee", // "it" - "iy": "ee", // "eat" - "ow": "ou", // "show" - "oy": "ou", // "toy" - "uh": "ou", // "book" - "uw": "ou", // "blue" - - // Consonants - "b": "mbp", // "bat" - "ch": "ch", // "chair" - "d": "td", // "dog" - "dh": "th", // "the" - "f": "fv", // "fish" - "g": "gk", // "go" - "hh": "aa", // "hat" - "jh": "ch", // "joy" - "k": "gk", // "cat" - "l": "aa", // "let" - "m": "mbp", // "mat" - "n": "aa", // "not" - "ng": "gk", // "sing" - "p": "mbp", // "pat" - "r": "aa", // "red" - "s": "s", // "sat" - "sh": "ch", // "ship" - "t": "td", // "top" - "th": "th", // "think" - "v": "fv", // "vat" - "w": "ou", // "wet" - "y": "ee", // "yet" - "z": "s", // "zoo" - "zh": "ch", // "measure" -} - -// GetVisemeForPhoneme returns the viseme for a phoneme -func GetVisemeForPhoneme(phoneme string) string { - if viseme, ok := VisemeMapping[phoneme]; ok { - return viseme - } - return "aa" // Default -} - -// PhonemeToVisemeTimeline converts phoneme timings to viseme timeline -func PhonemeToVisemeTimeline(phonemes []PhonemeTiming) []VisemeEvent { - if len(phonemes) == 0 { - return []VisemeEvent{} - } - - var visemes []VisemeEvent - currentViseme := GetVisemeForPhoneme(phonemes[0].Phoneme) - startTime := phonemes[0].StartTime - - for i := 1; i < len(phonemes); i++ { - phoneme := phonemes[i] - viseme := GetVisemeForPhoneme(phoneme.Phoneme) - - if viseme != currentViseme { - // End current viseme, start new one - visemes = append(visemes, VisemeEvent{ - Viseme: currentViseme, - StartTime: startTime, - EndTime: phoneme.StartTime, - }) - currentViseme = viseme - startTime = phoneme.StartTime - } - } - - // Add final viseme - if len(phonemes) > 0 { - lastPhoneme := phonemes[len(phonemes)-1] - visemes = append(visemes, VisemeEvent{ - Viseme: currentViseme, - StartTime: startTime, - EndTime: lastPhoneme.EndTime, - }) - } - - return visemes -} - -// PhonemeTiming represents a phoneme with timing -type PhonemeTiming struct { - Phoneme string - StartTime float64 - EndTime float64 -} - -// VisemeEvent represents a viseme event -type VisemeEvent struct { - Viseme string - StartTime float64 - EndTime float64 -} - diff --git a/virtual-banker/avatar/renderer/service.go b/virtual-banker/avatar/renderer/service.go deleted file mode 100644 index 95b81f3..0000000 --- a/virtual-banker/avatar/renderer/service.go +++ /dev/null @@ -1,143 +0,0 @@ -package renderer - -import ( - "context" - "fmt" -) - -// Service controls Unreal Engine avatar rendering -type Service interface { - StartSession(ctx context.Context, sessionID string) error - StopSession(ctx context.Context, sessionID string) error - SendAnimationParams(ctx context.Context, sessionID string, params *AnimationParams) error - GetVideoStream(ctx context.Context, sessionID string) (string, error) // Returns WebRTC stream URL -} - -// AnimationParams contains animation parameters for the avatar -type AnimationParams struct { - Visemes []VisemeEvent - Expressions *ExpressionParams - Gestures []GestureEvent - Gaze *GazeParams -} - -// VisemeEvent represents a viseme (lip shape) event -type VisemeEvent struct { - Viseme string - StartTime float64 - EndTime float64 - Intensity float64 -} - -// ExpressionParams contains facial expression parameters -type ExpressionParams struct { - Valence float64 // -1.0 to 1.0 - Arousal float64 // 0.0 to 1.0 - Emotion string // e.g., "happy", "neutral", "concerned" -} - -// GestureEvent represents a gesture event -type GestureEvent struct { - Type string // e.g., "nod", "point", "wave" - StartTime float64 - Duration float64 - Intensity float64 -} - -// GazeParams contains gaze/head tracking parameters -type GazeParams struct { - TargetX float64 - TargetY float64 - TargetZ float64 -} - -// PixelStreamingService implements avatar rendering using Unreal PixelStreaming -type PixelStreamingService struct { - unrealInstances map[string]*UnrealInstance -} - -// UnrealInstance represents a running Unreal Engine instance -type UnrealInstance struct { - SessionID string - ProcessID int - StreamURL string - Status string // "starting", "running", "stopping", "stopped" -} - -// NewPixelStreamingService creates a new PixelStreaming service -func NewPixelStreamingService() *PixelStreamingService { - return &PixelStreamingService{ - unrealInstances: make(map[string]*UnrealInstance), - } -} - -// StartSession starts an Unreal instance for a session -func (s *PixelStreamingService) StartSession(ctx context.Context, sessionID string) error { - // TODO: Launch Unreal Engine with PixelStreaming enabled - // This would involve: - // 1. Starting Unreal Engine process with command-line args for PixelStreaming - // 2. Configuring the instance for the session - // 3. Getting the WebRTC stream URL - // 4. Storing instance info - - instance := &UnrealInstance{ - SessionID: sessionID, - Status: "starting", - } - - s.unrealInstances[sessionID] = instance - - // Simulate instance startup - instance.Status = "running" - instance.StreamURL = fmt.Sprintf("ws://localhost:8888/stream/%s", sessionID) - - return nil -} - -// StopSession stops an Unreal instance -func (s *PixelStreamingService) StopSession(ctx context.Context, sessionID string) error { - instance, ok := s.unrealInstances[sessionID] - if !ok { - return fmt.Errorf("instance not found for session: %s", sessionID) - } - - instance.Status = "stopping" - // TODO: Terminate Unreal Engine process - instance.Status = "stopped" - delete(s.unrealInstances, sessionID) - - return nil -} - -// SendAnimationParams sends animation parameters to Unreal -func (s *PixelStreamingService) SendAnimationParams(ctx context.Context, sessionID string, params *AnimationParams) error { - instance, ok := s.unrealInstances[sessionID] - if !ok { - return fmt.Errorf("instance not found for session: %s", sessionID) - } - - // TODO: Send parameters via WebSocket or HTTP to Unreal PixelStreaming plugin - // This would involve: - // 1. Serializing AnimationParams to JSON - // 2. Sending to Unreal instance's control endpoint - // 3. Unreal receives and applies to avatar - - _ = instance // Use instance - - return nil -} - -// GetVideoStream returns the WebRTC stream URL for a session -func (s *PixelStreamingService) GetVideoStream(ctx context.Context, sessionID string) (string, error) { - instance, ok := s.unrealInstances[sessionID] - if !ok { - return "", fmt.Errorf("instance not found for session: %s", sessionID) - } - - if instance.Status != "running" { - return "", fmt.Errorf("instance not running for session: %s", sessionID) - } - - return instance.StreamURL, nil -} - diff --git a/virtual-banker/avatar/unreal/README.md b/virtual-banker/avatar/unreal/README.md deleted file mode 100644 index ac5fa4b..0000000 --- a/virtual-banker/avatar/unreal/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Unreal Engine Avatar Setup - -This directory contains the Unreal Engine project for the Virtual Banker avatar. - -## Prerequisites - -- Unreal Engine 5.3+ (or 5.4+ recommended) -- PixelStreaming plugin enabled -- Digital human character asset (Ready Player Me, MetaHuman, or custom) - -## Setup Instructions - -### 1. Create Unreal Project - -1. Open Unreal Engine Editor -2. Create new project: - - Template: Blank - - Blueprint or C++: Blueprint (or C++ if custom code needed) - - Target Platform: Desktop - - Quality: Maximum - - Raytracing: Enabled (optional, for better quality) - -### 2. Enable PixelStreaming - -1. Edit → Plugins -2. Search for "Pixel Streaming" -3. Enable the plugin -4. Restart Unreal Editor - -### 3. Import Digital Human - -1. Import your digital human character: - - Ready Player Me: Use their Unreal plugin - - MetaHuman: Use MetaHuman Creator - - Custom: Import FBX/glTF with blendshapes - -2. Set up blendshapes for visemes: - - Import viseme blendshapes (aa, ee, oh, ou, mbp, etc.) - - Map to animation system - -### 4. Configure PixelStreaming - -1. Edit → Project Settings → Plugins → Pixel Streaming -2. Configure: - - Streamer Port: 8888 - - WebRTC Port Range: 8888-8897 - - Enable WebRTC - -### 5. Set Up Animation Blueprint - -1. Create Animation Blueprint for avatar -2. Set up state machine: - - Idle - - Speaking (viseme-driven) - - Gesturing - - Expressions - -3. Connect viseme blendshapes to animation graph - -### 6. Create Control Blueprint - -1. Create Blueprint Actor for avatar control -2. Add functions: - - SetVisemes(VisemeData) - - SetExpression(Valence, Arousal) - - SetGesture(GestureType) - - SetGaze(Target) - -### 7. Build and Package - -1. Package project for Linux (for server deployment): - - File → Package Project → Linux - - Or use command line: - ``` - UnrealEditor-Cmd.exe -run=UnrealVersionSelector -project="path/to/project.uproject" -game -cook -package -build - ``` - -## Deployment - -The packaged Unreal project should be deployed to a GPU-enabled server with: -- NVIDIA GPU (RTX 3090+ recommended) -- CUDA drivers -- Sufficient VRAM (8GB+ per instance) - -## Integration - -The renderer service (`avatar/renderer/service.go`) controls Unreal instances via: -- Process management (start/stop instances) -- WebSocket communication (animation parameters) -- PixelStreaming WebRTC streams - -## Notes - -- Each active session requires one Unreal instance -- GPU resources should be allocated per instance -- Consider using Unreal's multi-instance support for scaling - diff --git a/virtual-banker/backend/api/realtime.go b/virtual-banker/backend/api/realtime.go deleted file mode 100644 index fd5833d..0000000 --- a/virtual-banker/backend/api/realtime.go +++ /dev/null @@ -1,35 +0,0 @@ -package api - -import ( - "net/http" - - "github.com/gorilla/mux" -) - -// HandleRealtimeWebSocket handles WebSocket upgrade for realtime communication -func (s *Server) HandleRealtimeWebSocket(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - sessionID := vars["id"] - - if sessionID == "" { - writeError(w, http.StatusBadRequest, "session_id is required", nil) - return - } - - // Get session to validate - _, err := s.sessionManager.GetSession(r.Context(), sessionID) - if err != nil { - writeError(w, http.StatusUnauthorized, "invalid session", err) - return - } - - // Upgrade to WebSocket - if s.realtimeGateway != nil { - if err := s.realtimeGateway.HandleWebSocket(w, r, sessionID); err != nil { - writeError(w, http.StatusInternalServerError, "failed to upgrade connection", err) - return - } - return - } - writeError(w, http.StatusServiceUnavailable, "realtime gateway not available", nil) -} diff --git a/virtual-banker/backend/api/routes.go b/virtual-banker/backend/api/routes.go deleted file mode 100644 index ab769a9..0000000 --- a/virtual-banker/backend/api/routes.go +++ /dev/null @@ -1,185 +0,0 @@ -package api - -import ( - "encoding/json" - "net/http" - "time" - - "github.com/explorer/virtual-banker/backend/realtime" - "github.com/explorer/virtual-banker/backend/session" - "github.com/gorilla/mux" -) - -// Server handles HTTP requests -type Server struct { - sessionManager *session.Manager - realtimeGateway *realtime.Gateway - router *mux.Router -} - -// NewServer creates a new API server -func NewServer(sessionManager *session.Manager, realtimeGateway *realtime.Gateway) *Server { - s := &Server{ - sessionManager: sessionManager, - realtimeGateway: realtimeGateway, - router: mux.NewRouter(), - } - s.setupRoutes() - return s -} - -// setupRoutes sets up all API routes -func (s *Server) setupRoutes() { - api := s.router.PathPrefix("/v1").Subrouter() - - // Session routes - api.HandleFunc("/sessions", s.handleCreateSession).Methods("POST") - api.HandleFunc("/sessions/{id}/refresh-token", s.handleRefreshToken).Methods("POST") - api.HandleFunc("/sessions/{id}/end", s.handleEndSession).Methods("POST") - - // Realtime WebSocket - api.HandleFunc("/realtime/{id}", s.HandleRealtimeWebSocket) - - // Health check - s.router.HandleFunc("/health", s.handleHealth).Methods("GET") -} - -// CreateSessionRequest represents a session creation request -type CreateSessionRequest struct { - TenantID string `json:"tenant_id"` - UserID string `json:"user_id"` - AuthAssertion string `json:"auth_assertion"` - PortalContext map[string]interface{} `json:"portal_context,omitempty"` -} - -// CreateSessionResponse represents a session creation response -type CreateSessionResponse struct { - SessionID string `json:"session_id"` - EphemeralToken string `json:"ephemeral_token"` - Config *session.TenantConfig `json:"config"` - ExpiresAt time.Time `json:"expires_at"` -} - -// handleCreateSession handles POST /v1/sessions -func (s *Server) handleCreateSession(w http.ResponseWriter, r *http.Request) { - var req CreateSessionRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body", err) - return - } - - if req.TenantID == "" || req.UserID == "" || req.AuthAssertion == "" { - writeError(w, http.StatusBadRequest, "tenant_id, user_id, and auth_assertion are required", nil) - return - } - - sess, err := s.sessionManager.CreateSession(r.Context(), req.TenantID, req.UserID, req.AuthAssertion) - if err != nil { - writeError(w, http.StatusInternalServerError, "failed to create session", err) - return - } - - resp := CreateSessionResponse{ - SessionID: sess.ID, - EphemeralToken: sess.EphemeralToken, - Config: sess.Config, - ExpiresAt: sess.ExpiresAt, - } - - writeJSON(w, http.StatusCreated, resp) -} - -// RefreshTokenResponse represents a token refresh response -type RefreshTokenResponse struct { - EphemeralToken string `json:"ephemeral_token"` - ExpiresAt time.Time `json:"expires_at"` -} - -// handleRefreshToken handles POST /v1/sessions/:id/refresh-token -func (s *Server) handleRefreshToken(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - sessionID := vars["id"] - - if sessionID == "" { - writeError(w, http.StatusBadRequest, "session_id is required", nil) - return - } - - newToken, err := s.sessionManager.RefreshToken(r.Context(), sessionID) - if err != nil { - if err.Error() == "session expired" { - writeError(w, http.StatusUnauthorized, "session expired", err) - return - } - writeError(w, http.StatusInternalServerError, "failed to refresh token", err) - return - } - - sess, err := s.sessionManager.GetSession(r.Context(), sessionID) - if err != nil { - writeError(w, http.StatusInternalServerError, "failed to get session", err) - return - } - - resp := RefreshTokenResponse{ - EphemeralToken: newToken, - ExpiresAt: sess.ExpiresAt, - } - - writeJSON(w, http.StatusOK, resp) -} - -// handleEndSession handles POST /v1/sessions/:id/end -func (s *Server) handleEndSession(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - sessionID := vars["id"] - - if sessionID == "" { - writeError(w, http.StatusBadRequest, "session_id is required", nil) - return - } - - if err := s.sessionManager.EndSession(r.Context(), sessionID); err != nil { - writeError(w, http.StatusInternalServerError, "failed to end session", err) - return - } - - writeJSON(w, http.StatusOK, map[string]string{"status": "ended"}) -} - -// handleHealth handles GET /health -func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]string{"status": "healthy"}) -} - -// ServeHTTP implements http.Handler -func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.router.ServeHTTP(w, r) -} - -// writeJSON writes a JSON response -func writeJSON(w http.ResponseWriter, status int, data interface{}) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - json.NewEncoder(w).Encode(data) -} - -// ErrorResponse represents an error response -type ErrorResponse struct { - Error string `json:"error"` - Message string `json:"message,omitempty"` -} - -// writeError writes an error response -func writeError(w http.ResponseWriter, status int, message string, err error) { - resp := ErrorResponse{ - Error: message, - Message: func() string { - if err != nil { - return err.Error() - } - return "" - }(), - } - writeJSON(w, status, resp) -} diff --git a/virtual-banker/backend/asr/service.go b/virtual-banker/backend/asr/service.go deleted file mode 100644 index d11ecee..0000000 --- a/virtual-banker/backend/asr/service.go +++ /dev/null @@ -1,102 +0,0 @@ -package asr - -import ( - "context" - "fmt" - "io" - "time" -) - -// Service provides speech-to-text functionality -type Service interface { - TranscribeStream(ctx context.Context, audioStream io.Reader) (<-chan TranscriptEvent, error) - Transcribe(ctx context.Context, audioData []byte) (string, error) -} - -// TranscriptEvent represents a transcription event -type TranscriptEvent struct { - Type string `json:"type"` // "partial" or "final" - Text string `json:"text"` - Confidence float64 `json:"confidence,omitempty"` - Timestamp int64 `json:"timestamp"` - Words []Word `json:"words,omitempty"` -} - -// Word represents a word with timing information -type Word struct { - Word string `json:"word"` - StartTime float64 `json:"start_time"` - EndTime float64 `json:"end_time"` - Confidence float64 `json:"confidence,omitempty"` -} - -// MockASRService is a mock implementation for development -type MockASRService struct{} - -// NewMockASRService creates a new mock ASR service -func NewMockASRService() *MockASRService { - return &MockASRService{} -} - -// TranscribeStream transcribes an audio stream -func (s *MockASRService) TranscribeStream(ctx context.Context, audioStream io.Reader) (<-chan TranscriptEvent, error) { - events := make(chan TranscriptEvent, 10) - - go func() { - defer close(events) - - // Mock implementation - in production, integrate with Deepgram, Google STT, etc. - // For now, just send a mock event - select { - case <-ctx.Done(): - return - case events <- TranscriptEvent{ - Type: "final", - Text: "Hello, how can I help you today?", - Confidence: 0.95, - Timestamp: time.Now().Unix(), - }: - } - }() - - return events, nil -} - -// Transcribe transcribes audio data -func (s *MockASRService) Transcribe(ctx context.Context, audioData []byte) (string, error) { - // Mock implementation - return "Hello, how can I help you today?", nil -} - -// DeepgramASRService integrates with Deepgram (example - requires API key) -type DeepgramASRService struct { - apiKey string -} - -// NewDeepgramASRService creates a new Deepgram ASR service -func NewDeepgramASRService(apiKey string) *DeepgramASRService { - return &DeepgramASRService{ - apiKey: apiKey, - } -} - -// TranscribeStream transcribes using Deepgram streaming API -func (s *DeepgramASRService) TranscribeStream(ctx context.Context, audioStream io.Reader) (<-chan TranscriptEvent, error) { - events := make(chan TranscriptEvent, 10) - - // TODO: Implement Deepgram streaming API integration - // This would involve: - // 1. Establishing WebSocket connection to Deepgram - // 2. Sending audio chunks - // 3. Receiving partial and final transcripts - // 4. Converting to TranscriptEvent format - - return events, fmt.Errorf("not implemented - requires Deepgram API integration") -} - -// Transcribe transcribes using Deepgram REST API -func (s *DeepgramASRService) Transcribe(ctx context.Context, audioData []byte) (string, error) { - // TODO: Implement Deepgram REST API integration - return "", fmt.Errorf("not implemented - requires Deepgram API integration") -} - diff --git a/virtual-banker/backend/go.mod b/virtual-banker/backend/go.mod deleted file mode 100644 index c33e324..0000000 --- a/virtual-banker/backend/go.mod +++ /dev/null @@ -1,22 +0,0 @@ -module github.com/explorer/virtual-banker/backend - -go 1.21 - -require ( - github.com/gorilla/mux v1.8.1 - github.com/gorilla/websocket v1.5.1 - github.com/jackc/pgx/v5 v5.5.1 - github.com/redis/go-redis/v9 v9.3.0 -) - -require ( - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/text v0.14.0 // indirect -) diff --git a/virtual-banker/backend/go.sum b/virtual-banker/backend/go.sum deleted file mode 100644 index aa51ef9..0000000 --- a/virtual-banker/backend/go.sum +++ /dev/null @@ -1,44 +0,0 @@ -github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.1 h1:5I9etrGkLrN+2XPCsi6XLlV5DITbSL/xBZdmAxFcXPI= -github.com/jackc/pgx/v5 v5.5.1/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= -github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/virtual-banker/backend/llm/gateway.go b/virtual-banker/backend/llm/gateway.go deleted file mode 100644 index 22f6e8b..0000000 --- a/virtual-banker/backend/llm/gateway.go +++ /dev/null @@ -1,102 +0,0 @@ -package llm - -import ( - "context" - "fmt" -) - -// Gateway provides LLM functionality -type Gateway interface { - Generate(ctx context.Context, prompt string, options *GenerateOptions) (*GenerateResponse, error) -} - -// GenerateOptions contains options for generation -type GenerateOptions struct { - Temperature float64 - MaxTokens int - Tools []Tool - TenantID string - UserID string - ConversationHistory []Message -} - -// Tool represents a callable tool/function -type Tool struct { - Name string - Description string - Parameters map[string]interface{} -} - -// Message represents a conversation message -type Message struct { - Role string // "user" or "assistant" - Content string -} - -// GenerateResponse contains the LLM response -type GenerateResponse struct { - Text string - Tools []ToolCall - Emotion *Emotion - Gestures []string -} - -// ToolCall represents a tool call request -type ToolCall struct { - Name string - Arguments map[string]interface{} -} - -// Emotion represents emotional state for avatar -type Emotion struct { - Valence float64 // -1.0 to 1.0 - Arousal float64 // 0.0 to 1.0 -} - -// MockLLMGateway is a mock implementation for development -type MockLLMGateway struct{} - -// NewMockLLMGateway creates a new mock LLM gateway -func NewMockLLMGateway() *MockLLMGateway { - return &MockLLMGateway{} -} - -// Generate generates a response using mock LLM -func (g *MockLLMGateway) Generate(ctx context.Context, prompt string, options *GenerateOptions) (*GenerateResponse, error) { - // Mock implementation - return &GenerateResponse{ - Text: "I understand. How can I assist you with your banking needs today?", - Emotion: &Emotion{ - Valence: 0.5, - Arousal: 0.3, - }, - Gestures: []string{"nod"}, - }, nil -} - -// OpenAIGateway integrates with OpenAI (example - requires API key) -type OpenAIGateway struct { - apiKey string - model string -} - -// NewOpenAIGateway creates a new OpenAI gateway -func NewOpenAIGateway(apiKey, model string) *OpenAIGateway { - return &OpenAIGateway{ - apiKey: apiKey, - model: model, - } -} - -// Generate generates using OpenAI API -func (g *OpenAIGateway) Generate(ctx context.Context, prompt string, options *GenerateOptions) (*GenerateResponse, error) { - // TODO: Implement OpenAI API integration - // This would involve: - // 1. Building the prompt with system message, conversation history - // 2. Adding tool definitions if tools are provided - // 3. Making API call to OpenAI - // 4. Parsing response and extracting tool calls - // 5. Mapping to GenerateResponse format - return nil, fmt.Errorf("not implemented - requires OpenAI API integration") -} - diff --git a/virtual-banker/backend/llm/prompt.go b/virtual-banker/backend/llm/prompt.go deleted file mode 100644 index a9da32a..0000000 --- a/virtual-banker/backend/llm/prompt.go +++ /dev/null @@ -1,124 +0,0 @@ -package llm - -import ( - "fmt" - "strings" -) - -// BuildPrompt builds a prompt from components -func BuildPrompt(tenantConfig *TenantConfig, conversationHistory []Message, userInput string, retrievedDocs []RetrievedDoc) string { - var parts []string - - // System message - systemMsg := buildSystemMessage(tenantConfig) - parts = append(parts, systemMsg) - - // Retrieved documents (RAG context) - if len(retrievedDocs) > 0 { - parts = append(parts, "\n## Context:") - for i, doc := range retrievedDocs { - parts = append(parts, fmt.Sprintf("\n[Document %d]", i+1)) - parts = append(parts, fmt.Sprintf("Title: %s", doc.Title)) - parts = append(parts, fmt.Sprintf("Content: %s", doc.Content)) - if doc.URL != "" { - parts = append(parts, fmt.Sprintf("Source: %s", doc.URL)) - } - } - } - - // Conversation history - if len(conversationHistory) > 0 { - parts = append(parts, "\n## Conversation History:") - for _, msg := range conversationHistory { - parts = append(parts, fmt.Sprintf("%s: %s", strings.Title(msg.Role), msg.Content)) - } - } - - // Current user input - parts = append(parts, fmt.Sprintf("\n## User: %s", userInput)) - parts = append(parts, "\n## Assistant:") - - return strings.Join(parts, "\n") -} - -// TenantConfig holds tenant-specific configuration -type TenantConfig struct { - Greeting string - Tone string // "professional", "friendly", "formal" - Disclaimers []string - AllowedTools []string -} - -// RetrievedDoc represents a retrieved document from RAG -type RetrievedDoc struct { - Title string - Content string - URL string - Score float64 -} - -// BuildPromptWithRAG builds a prompt with RAG context -func BuildPromptWithRAG(tenantConfig *TenantConfig, conversationHistory []Message, userInput string, retrievedDocs []RetrievedDoc) string { - var parts []string - - // System message - systemMsg := buildSystemMessage(tenantConfig) - parts = append(parts, systemMsg) - - // Retrieved documents (RAG context) - if len(retrievedDocs) > 0 { - parts = append(parts, "\n## Context:") - for i, doc := range retrievedDocs { - parts = append(parts, fmt.Sprintf("\n[Document %d]", i+1)) - parts = append(parts, fmt.Sprintf("Title: %s", doc.Title)) - parts = append(parts, fmt.Sprintf("Content: %s", doc.Content)) - if doc.URL != "" { - parts = append(parts, fmt.Sprintf("Source: %s", doc.URL)) - } - } - } - - // Conversation history - if len(conversationHistory) > 0 { - parts = append(parts, "\n## Conversation History:") - for _, msg := range conversationHistory { - parts = append(parts, fmt.Sprintf("%s: %s", strings.Title(msg.Role), msg.Content)) - } - } - - // Current user input - parts = append(parts, fmt.Sprintf("\n## User: %s", userInput)) - parts = append(parts, "\n## Assistant:") - - return strings.Join(parts, "\n") -} - -// buildSystemMessage builds the system message -func buildSystemMessage(config *TenantConfig) string { - var parts []string - - parts = append(parts, "You are a helpful Virtual Banker assistant.") - - if config.Tone != "" { - parts = append(parts, fmt.Sprintf("Your tone should be %s.", config.Tone)) - } - - if len(config.Disclaimers) > 0 { - parts = append(parts, "\nImportant disclaimers:") - for _, disclaimer := range config.Disclaimers { - parts = append(parts, fmt.Sprintf("- %s", disclaimer)) - } - } - - if len(config.AllowedTools) > 0 { - parts = append(parts, "\nYou have access to the following tools:") - for _, tool := range config.AllowedTools { - parts = append(parts, fmt.Sprintf("- %s", tool)) - } - } - - parts = append(parts, "\nAlways be helpful, accurate, and respectful.") - parts = append(parts, "If you don't know something, say so and offer to help find the answer.") - - return strings.Join(parts, "\n") -} diff --git a/virtual-banker/backend/main.go b/virtual-banker/backend/main.go deleted file mode 100644 index a00983c..0000000 --- a/virtual-banker/backend/main.go +++ /dev/null @@ -1,136 +0,0 @@ -package main - -import ( - "context" - "log" - "net/http" - "os" - "os/signal" - "syscall" - "time" - - "github.com/explorer/virtual-banker/backend/api" - "github.com/explorer/virtual-banker/backend/asr" - "github.com/explorer/virtual-banker/backend/llm" - "github.com/explorer/virtual-banker/backend/orchestrator" - "github.com/explorer/virtual-banker/backend/rag" - "github.com/explorer/virtual-banker/backend/realtime" - "github.com/explorer/virtual-banker/backend/session" - "github.com/explorer/virtual-banker/backend/tools" - "github.com/explorer/virtual-banker/backend/tools/banking" - "github.com/explorer/virtual-banker/backend/tts" - "github.com/jackc/pgx/v5/pgxpool" - "github.com/redis/go-redis/v9" -) - -func main() { - // Load configuration from environment - dbURL := getEnv("DATABASE_URL", "postgres://explorer:changeme@localhost:5432/explorer?sslmode=disable") - redisURL := getEnv("REDIS_URL", "redis://localhost:6379") - port := getEnv("PORT", "8081") - - // Initialize database connection - db, err := pgxpool.New(context.Background(), dbURL) - if err != nil { - log.Fatalf("Failed to connect to database: %v", err) - } - defer db.Close() - - // Initialize Redis connection - opt, err := redis.ParseURL(redisURL) - if err != nil { - log.Fatalf("Failed to parse Redis URL: %v", err) - } - redisClient := redis.NewClient(opt) - defer redisClient.Close() - - // Test connections - if err := db.Ping(context.Background()); err != nil { - log.Fatalf("Database ping failed: %v", err) - } - if err := redisClient.Ping(context.Background()).Err(); err != nil { - log.Fatalf("Redis ping failed: %v", err) - } - - // Initialize services - sessionManager := session.NewManager(db, redisClient) - - // Initialize ASR/TTS (using mocks for now) - asrService := asr.NewMockASRService() - ttsService := tts.NewMockTTSService() - - // Initialize LLM (using mock for now) - llmGateway := llm.NewMockLLMGateway() - - // Initialize RAG - ragService := rag.NewRAGService(db) - - // Initialize tools - toolRegistry := tools.NewRegistry() - toolRegistry.Register(banking.NewAccountStatusTool()) - toolRegistry.Register(banking.NewCreateTicketTool()) - toolRegistry.Register(banking.NewScheduleAppointmentTool()) - toolRegistry.Register(banking.NewSubmitPaymentTool()) - - auditLogger := &tools.MockAuditLogger{} - toolExecutor := tools.NewExecutor(toolRegistry, auditLogger) - - // Initialize orchestrator - convOrchestrator := orchestrator.NewOrchestrator( - asrService, - ttsService, - llmGateway, - ragService, - toolExecutor, - ) - - // Initialize realtime gateway - realtimeGateway := realtime.NewGateway() - - // Initialize API server - apiServer := api.NewServer(sessionManager, realtimeGateway) - - // Store orchestrator reference (would be used by handlers) - _ = convOrchestrator - - // Create HTTP server - srv := &http.Server{ - Addr: ":" + port, - Handler: apiServer, - ReadTimeout: 15 * time.Second, - WriteTimeout: 15 * time.Second, - IdleTimeout: 60 * time.Second, - } - - // Start server in goroutine - go func() { - log.Printf("Virtual Banker API server starting on port %s", port) - if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("Server failed to start: %v", err) - } - }() - - // Wait for interrupt signal - quit := make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - <-quit - - log.Println("Shutting down server...") - - // Graceful shutdown - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := srv.Shutdown(ctx); err != nil { - log.Fatalf("Server forced to shutdown: %v", err) - } - - log.Println("Server exited") -} - -func getEnv(key, defaultValue string) string { - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue -} diff --git a/virtual-banker/backend/memory/service.go b/virtual-banker/backend/memory/service.go deleted file mode 100644 index bd74cf4..0000000 --- a/virtual-banker/backend/memory/service.go +++ /dev/null @@ -1,163 +0,0 @@ -package memory - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/jackc/pgx/v5/pgxpool" -) - -// Service manages user memory and preferences -type Service interface { - GetProfile(ctx context.Context, userID, tenantID string) (*UserProfile, error) - SaveProfile(ctx context.Context, profile *UserProfile) error - GetHistory(ctx context.Context, userID, tenantID string, limit int) ([]ConversationHistory, error) - SaveHistory(ctx context.Context, history *ConversationHistory) error -} - -// UserProfile represents user preferences and memory -type UserProfile struct { - UserID string - TenantID string - Preferences map[string]interface{} - Context map[string]interface{} - CreatedAt string - UpdatedAt string -} - -// ConversationHistory represents a conversation history entry -type ConversationHistory struct { - ID string - UserID string - TenantID string - SessionID string - Messages []Message - CreatedAt string -} - -// Message represents a message in history -type Message struct { - Role string - Content string - Timestamp string -} - -// MemoryService implements memory using PostgreSQL -type MemoryService struct { - db *pgxpool.Pool -} - -// NewMemoryService creates a new memory service -func NewMemoryService(db *pgxpool.Pool) *MemoryService { - return &MemoryService{ - db: db, - } -} - -// GetProfile gets user profile -func (s *MemoryService) GetProfile(ctx context.Context, userID, tenantID string) (*UserProfile, error) { - query := ` - SELECT user_id, tenant_id, preferences, context, created_at, updated_at - FROM user_profiles - WHERE user_id = $1 AND tenant_id = $2 - ` - - var profile UserProfile - var prefsJSON, contextJSON []byte - - err := s.db.QueryRow(ctx, query, userID, tenantID).Scan( - &profile.UserID, - &profile.TenantID, - &prefsJSON, - &contextJSON, - &profile.CreatedAt, - &profile.UpdatedAt, - ) - if err != nil { - // Return default profile if not found - return &UserProfile{ - UserID: userID, - TenantID: tenantID, - Preferences: make(map[string]interface{}), - Context: make(map[string]interface{}), - }, nil - } - - if err := json.Unmarshal(prefsJSON, &profile.Preferences); err != nil { - profile.Preferences = make(map[string]interface{}) - } - if err := json.Unmarshal(contextJSON, &profile.Context); err != nil { - profile.Context = make(map[string]interface{}) - } - - return &profile, nil -} - -// SaveProfile saves user profile -func (s *MemoryService) SaveProfile(ctx context.Context, profile *UserProfile) error { - prefsJSON, _ := json.Marshal(profile.Preferences) - contextJSON, _ := json.Marshal(profile.Context) - - query := ` - INSERT INTO user_profiles (user_id, tenant_id, preferences, context, created_at, updated_at) - VALUES ($1, $2, $3, $4, NOW(), NOW()) - ON CONFLICT (user_id, tenant_id) DO UPDATE SET - preferences = $3, - context = $4, - updated_at = NOW() - ` - - _, err := s.db.Exec(ctx, query, profile.UserID, profile.TenantID, prefsJSON, contextJSON) - return err -} - -// GetHistory gets conversation history -func (s *MemoryService) GetHistory(ctx context.Context, userID, tenantID string, limit int) ([]ConversationHistory, error) { - if limit <= 0 { - limit = 10 - } - - query := ` - SELECT id, user_id, tenant_id, session_id, messages, created_at - FROM conversation_history - WHERE user_id = $1 AND tenant_id = $2 - ORDER BY created_at DESC - LIMIT $3 - ` - - rows, err := s.db.Query(ctx, query, userID, tenantID, limit) - if err != nil { - return nil, fmt.Errorf("failed to query: %w", err) - } - defer rows.Close() - - var histories []ConversationHistory - for rows.Next() { - var history ConversationHistory - var messagesJSON []byte - if err := rows.Scan(&history.ID, &history.UserID, &history.TenantID, &history.SessionID, &messagesJSON, &history.CreatedAt); err != nil { - continue - } - if err := json.Unmarshal(messagesJSON, &history.Messages); err != nil { - history.Messages = []Message{} - } - histories = append(histories, history) - } - - return histories, nil -} - -// SaveHistory saves conversation history -func (s *MemoryService) SaveHistory(ctx context.Context, history *ConversationHistory) error { - messagesJSON, _ := json.Marshal(history.Messages) - - query := ` - INSERT INTO conversation_history (id, user_id, tenant_id, session_id, messages, created_at) - VALUES ($1, $2, $3, $4, $5, NOW()) - ` - - _, err := s.db.Exec(ctx, query, history.ID, history.UserID, history.TenantID, history.SessionID, messagesJSON) - return err -} - diff --git a/virtual-banker/backend/observability/metrics.go b/virtual-banker/backend/observability/metrics.go deleted file mode 100644 index 7b1c383..0000000 --- a/virtual-banker/backend/observability/metrics.go +++ /dev/null @@ -1,73 +0,0 @@ -package observability - -import ( - "sync/atomic" - "time" -) - -// Metrics collects system metrics -type Metrics struct { - SessionCreations int64 - ActiveSessions int64 - MessagesProcessed int64 - ASRLatency int64 // microseconds - TTSLatency int64 // microseconds - LLMLatency int64 // microseconds - Errors int64 -} - -var globalMetrics = &Metrics{} - -// GetMetrics returns current metrics -func GetMetrics() *Metrics { - return &Metrics{ - SessionCreations: atomic.LoadInt64(&globalMetrics.SessionCreations), - ActiveSessions: atomic.LoadInt64(&globalMetrics.ActiveSessions), - MessagesProcessed: atomic.LoadInt64(&globalMetrics.MessagesProcessed), - ASRLatency: atomic.LoadInt64(&globalMetrics.ASRLatency), - TTSLatency: atomic.LoadInt64(&globalMetrics.TTSLatency), - LLMLatency: atomic.LoadInt64(&globalMetrics.LLMLatency), - Errors: atomic.LoadInt64(&globalMetrics.Errors), - } -} - -// IncrementSessionCreations increments session creation count -func IncrementSessionCreations() { - atomic.AddInt64(&globalMetrics.SessionCreations, 1) -} - -// IncrementActiveSessions increments active session count -func IncrementActiveSessions() { - atomic.AddInt64(&globalMetrics.ActiveSessions, 1) -} - -// DecrementActiveSessions decrements active session count -func DecrementActiveSessions() { - atomic.AddInt64(&globalMetrics.ActiveSessions, -1) -} - -// IncrementMessagesProcessed increments message count -func IncrementMessagesProcessed() { - atomic.AddInt64(&globalMetrics.MessagesProcessed, 1) -} - -// RecordASRLatency records ASR latency -func RecordASRLatency(duration time.Duration) { - atomic.StoreInt64(&globalMetrics.ASRLatency, duration.Microseconds()) -} - -// RecordTTSLatency records TTS latency -func RecordTTSLatency(duration time.Duration) { - atomic.StoreInt64(&globalMetrics.TTSLatency, duration.Microseconds()) -} - -// RecordLLMLatency records LLM latency -func RecordLLMLatency(duration time.Duration) { - atomic.StoreInt64(&globalMetrics.LLMLatency, duration.Microseconds()) -} - -// IncrementErrors increments error count -func IncrementErrors() { - atomic.AddInt64(&globalMetrics.Errors, 1) -} - diff --git a/virtual-banker/backend/observability/tracing.go b/virtual-banker/backend/observability/tracing.go deleted file mode 100644 index 73b0519..0000000 --- a/virtual-banker/backend/observability/tracing.go +++ /dev/null @@ -1,48 +0,0 @@ -package observability - -import ( - "context" - "fmt" -) - -// Tracer provides distributed tracing -type Tracer interface { - StartSpan(ctx context.Context, name string) (context.Context, Span) -} - -// Span represents a tracing span -type Span interface { - End() - SetAttribute(key string, value interface{}) - SetError(err error) -} - -// MockTracer is a mock tracer for development -type MockTracer struct{} - -// StartSpan starts a new span -func (t *MockTracer) StartSpan(ctx context.Context, name string) (context.Context, Span) { - return ctx, &MockSpan{} -} - -// MockSpan is a mock span -type MockSpan struct{} - -// End ends the span -func (m *MockSpan) End() {} - -// SetAttribute sets an attribute -func (m *MockSpan) SetAttribute(key string, value interface{}) {} - -// SetError sets an error -func (m *MockSpan) SetError(err error) {} - -// TraceConversation traces a conversation turn -func TraceConversation(ctx context.Context, tracer Tracer, sessionID, userID string, input string) (context.Context, Span) { - ctx, span := tracer.StartSpan(ctx, "conversation.turn") - span.SetAttribute("session_id", sessionID) - span.SetAttribute("user_id", userID) - span.SetAttribute("input_length", len(input)) - return ctx, span -} - diff --git a/virtual-banker/backend/orchestrator/orchestrator.go b/virtual-banker/backend/orchestrator/orchestrator.go deleted file mode 100644 index 11b40df..0000000 --- a/virtual-banker/backend/orchestrator/orchestrator.go +++ /dev/null @@ -1,284 +0,0 @@ -package orchestrator - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/explorer/virtual-banker/backend/asr" - "github.com/explorer/virtual-banker/backend/llm" - "github.com/explorer/virtual-banker/backend/rag" - "github.com/explorer/virtual-banker/backend/tools" - "github.com/explorer/virtual-banker/backend/tts" -) - -// State represents the conversation state -type State string - -const ( - StateIdle State = "IDLE" - StateListening State = "LISTENING" - StateThinking State = "THINKING" - StateSpeaking State = "SPEAKING" -) - -// Orchestrator orchestrates conversation flow -type Orchestrator struct { - sessions map[string]*SessionOrchestrator - mu sync.RWMutex - asr asr.Service - tts tts.Service - llm llm.Gateway - rag rag.Service - tools *tools.Executor -} - -// NewOrchestrator creates a new orchestrator -func NewOrchestrator(asrService asr.Service, ttsService tts.Service, llmGateway llm.Gateway, ragService rag.Service, toolExecutor *tools.Executor) *Orchestrator { - return &Orchestrator{ - sessions: make(map[string]*SessionOrchestrator), - asr: asrService, - tts: ttsService, - llm: llmGateway, - rag: ragService, - tools: toolExecutor, - } -} - -// SessionOrchestrator manages a single session's conversation -type SessionOrchestrator struct { - sessionID string - tenantID string - userID string - state State - mu sync.RWMutex - ctx context.Context - cancel context.CancelFunc - asr asr.Service - tts tts.Service - llm llm.Gateway - rag rag.Service - tools *tools.Executor - conversation []llm.Message -} - -// GetOrCreateSession gets or creates a session orchestrator -func (o *Orchestrator) GetOrCreateSession(sessionID, tenantID, userID string) *SessionOrchestrator { - o.mu.RLock() - sess, ok := o.sessions[sessionID] - o.mu.RUnlock() - - if ok { - return sess - } - - o.mu.Lock() - defer o.mu.Unlock() - - // Double-check - if sess, ok := o.sessions[sessionID]; ok { - return sess - } - - ctx, cancel := context.WithCancel(context.Background()) - sess = &SessionOrchestrator{ - sessionID: sessionID, - tenantID: tenantID, - userID: userID, - state: StateIdle, - ctx: ctx, - cancel: cancel, - asr: o.asr, - tts: o.tts, - llm: o.llm, - rag: o.rag, - tools: o.tools, - conversation: []llm.Message{}, - } - - o.sessions[sessionID] = sess - return sess -} - -// ProcessAudio processes incoming audio -func (so *SessionOrchestrator) ProcessAudio(ctx context.Context, audioData []byte) error { - so.mu.Lock() - currentState := so.state - so.mu.Unlock() - - // Handle barge-in: if speaking, stop and switch to listening - if currentState == StateSpeaking { - so.StopSpeaking() - } - - so.SetState(StateListening) - - // Transcribe audio - transcript, err := so.asr.Transcribe(ctx, audioData) - if err != nil { - return fmt.Errorf("failed to transcribe: %w", err) - } - - // Process transcript - so.SetState(StateThinking) - response, err := so.processTranscript(ctx, transcript) - if err != nil { - return fmt.Errorf("failed to process transcript: %w", err) - } - - // Synthesize response - so.SetState(StateSpeaking) - return so.speak(ctx, response) -} - -// ProcessText processes incoming text message -func (so *SessionOrchestrator) ProcessText(ctx context.Context, text string) error { - so.SetState(StateThinking) - - // Process text - response, err := so.processTranscript(ctx, text) - if err != nil { - return fmt.Errorf("failed to process text: %w", err) - } - - // Synthesize response - so.SetState(StateSpeaking) - return so.speak(ctx, response) -} - -// processTranscript processes a transcript and generates a response -func (so *SessionOrchestrator) processTranscript(ctx context.Context, transcript string) (string, error) { - // Add user message to conversation - so.conversation = append(so.conversation, llm.Message{ - Role: "user", - Content: transcript, - }) - - // Retrieve relevant documents from RAG - var retrievedDocs []rag.RetrievedDoc - if so.rag != nil { - docs, err := so.rag.Retrieve(ctx, transcript, so.tenantID, 5) - if err == nil { - retrievedDocs = docs - } - } - - // Build prompt with RAG context - // Convert retrieved docs to LLM format - ragDocs := make([]llm.RetrievedDoc, len(retrievedDocs)) - for i, doc := range retrievedDocs { - ragDocs[i] = llm.RetrievedDoc{ - Title: doc.Title, - Content: doc.Content, - URL: doc.URL, - Score: doc.Score, - } - } - - // Get available tools (would come from tenant config) - availableTools := []llm.Tool{} // TODO: Get from tenant config - - // Call LLM - options := &llm.GenerateOptions{ - Temperature: 0.7, - MaxTokens: 500, - Tools: availableTools, - TenantID: so.tenantID, - UserID: so.userID, - ConversationHistory: so.conversation, - } - - response, err := so.llm.Generate(ctx, transcript, options) - if err != nil { - return "", fmt.Errorf("failed to generate response: %w", err) - } - - // Execute tool calls if any - if len(response.Tools) > 0 && so.tools != nil { - for _, toolCall := range response.Tools { - result, err := so.tools.Execute(ctx, toolCall.Name, toolCall.Arguments, so.userID, so.tenantID) - if err != nil { - // Log error but continue - fmt.Printf("Tool execution error: %v\n", err) - continue - } - - // Add tool result to conversation - if result.Success { - so.conversation = append(so.conversation, llm.Message{ - Role: "assistant", - Content: fmt.Sprintf("Tool %s executed successfully: %v", toolCall.Name, result.Data), - }) - } - } - } - - // Add assistant response to conversation - so.conversation = append(so.conversation, llm.Message{ - Role: "assistant", - Content: response.Text, - }) - - return response.Text, nil -} - -// speak synthesizes and plays audio -func (so *SessionOrchestrator) speak(ctx context.Context, text string) error { - // Synthesize audio - audioData, err := so.tts.Synthesize(ctx, text) - if err != nil { - return fmt.Errorf("failed to synthesize: %w", err) - } - - // Get visemes for avatar - visemes, err := so.tts.GetVisemes(ctx, text) - if err != nil { - // Log error but continue - fmt.Printf("Failed to get visemes: %v\n", err) - } - - // TODO: Send audio and visemes to client via WebRTC/WebSocket - _ = audioData - _ = visemes - - // Simulate speaking duration - time.Sleep(time.Duration(len(text)*50) * time.Millisecond) - - so.SetState(StateIdle) - return nil -} - -// StopSpeaking stops current speech (barge-in) -func (so *SessionOrchestrator) StopSpeaking() { - so.mu.Lock() - defer so.mu.Unlock() - - if so.state == StateSpeaking { - // Cancel current TTS synthesis - so.cancel() - ctx, cancel := context.WithCancel(context.Background()) - so.ctx = ctx - so.cancel = cancel - so.state = StateIdle - } -} - -// SetState sets the conversation state -func (so *SessionOrchestrator) SetState(state State) { - so.mu.Lock() - defer so.mu.Unlock() - so.state = state -} - -// GetState gets the current conversation state -func (so *SessionOrchestrator) GetState() State { - so.mu.RLock() - defer so.mu.RUnlock() - return so.state -} - -// Close closes the session orchestrator -func (so *SessionOrchestrator) Close() { - so.cancel() -} diff --git a/virtual-banker/backend/rag/service.go b/virtual-banker/backend/rag/service.go deleted file mode 100644 index 512a0f8..0000000 --- a/virtual-banker/backend/rag/service.go +++ /dev/null @@ -1,110 +0,0 @@ -package rag - -import ( - "context" - "fmt" - - "github.com/jackc/pgx/v5/pgxpool" -) - -// Service provides RAG (Retrieval-Augmented Generation) functionality -type Service interface { - Retrieve(ctx context.Context, query string, tenantID string, topK int) ([]RetrievedDoc, error) - Ingest(ctx context.Context, doc *Document) error -} - -// RetrievedDoc represents a retrieved document -type RetrievedDoc struct { - ID string - Title string - Content string - URL string - Score float64 -} - -// Document represents a document to be ingested -type Document struct { - ID string - TenantID string - Title string - Content string - URL string - Metadata map[string]interface{} -} - -// RAGService implements RAG using pgvector -type RAGService struct { - db *pgxpool.Pool -} - -// NewRAGService creates a new RAG service -func NewRAGService(db *pgxpool.Pool) *RAGService { - return &RAGService{ - db: db, - } -} - -// Retrieve retrieves relevant documents -func (s *RAGService) Retrieve(ctx context.Context, query string, tenantID string, topK int) ([]RetrievedDoc, error) { - if topK <= 0 { - topK = 5 - } - - // TODO: Generate embedding for query - // For now, use simple text search - querySQL := ` - SELECT id, title, content, metadata->>'url' as url, - ts_rank(to_tsvector('english', content), plainto_tsquery('english', $1)) as score - FROM knowledge_base - WHERE tenant_id = $2 - ORDER BY score DESC - LIMIT $3 - ` - - rows, err := s.db.Query(ctx, querySQL, query, tenantID, topK) - if err != nil { - return nil, fmt.Errorf("failed to query: %w", err) - } - defer rows.Close() - - var docs []RetrievedDoc - for rows.Next() { - var doc RetrievedDoc - var url *string - if err := rows.Scan(&doc.ID, &doc.Title, &doc.Content, &url, &doc.Score); err != nil { - continue - } - if url != nil { - doc.URL = *url - } - docs = append(docs, doc) - } - - return docs, nil -} - -// Ingest ingests a document into the knowledge base -func (s *RAGService) Ingest(ctx context.Context, doc *Document) error { - // TODO: Generate embedding for document content - // For now, just insert without embedding - query := ` - INSERT INTO knowledge_base (id, tenant_id, title, content, metadata) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (id) DO UPDATE SET - title = $3, - content = $4, - metadata = $5, - updated_at = NOW() - ` - - metadata := map[string]interface{}{ - "url": doc.URL, - } - for k, v := range doc.Metadata { - metadata[k] = v - } - - _, err := s.db.Exec(ctx, query, doc.ID, doc.TenantID, doc.Title, doc.Content, metadata) - return err -} - diff --git a/virtual-banker/backend/realtime/gateway.go b/virtual-banker/backend/realtime/gateway.go deleted file mode 100644 index adac344..0000000 --- a/virtual-banker/backend/realtime/gateway.go +++ /dev/null @@ -1,198 +0,0 @@ -package realtime - -import ( - "context" - "encoding/json" - "fmt" - "log" - "net/http" - "sync" - "time" - - "github.com/gorilla/websocket" -) - -var upgrader = websocket.Upgrader{ - CheckOrigin: func(r *http.Request) bool { - // In production, validate origin properly - return true - }, -} - -// Gateway handles WebRTC signaling and WebSocket connections -type Gateway struct { - connections map[string]*Connection - mu sync.RWMutex -} - -// NewGateway creates a new WebRTC gateway -func NewGateway() *Gateway { - return &Gateway{ - connections: make(map[string]*Connection), - } -} - -// Connection represents a WebSocket connection for signaling -type Connection struct { - sessionID string - ws *websocket.Conn - send chan []byte - ctx context.Context - cancel context.CancelFunc -} - -// HandleWebSocket handles WebSocket upgrade for signaling -func (g *Gateway) HandleWebSocket(w http.ResponseWriter, r *http.Request, sessionID string) error { - ws, err := upgrader.Upgrade(w, r, nil) - if err != nil { - return fmt.Errorf("failed to upgrade connection: %w", err) - } - - ctx, cancel := context.WithCancel(r.Context()) - conn := &Connection{ - sessionID: sessionID, - ws: ws, - send: make(chan []byte, 256), - ctx: ctx, - cancel: cancel, - } - - g.mu.Lock() - g.connections[sessionID] = conn - g.mu.Unlock() - - // Start goroutines - go conn.writePump() - go conn.readPump(g) - - return nil -} - -// SendMessage sends a message to a specific session -func (g *Gateway) SendMessage(sessionID string, message interface{}) error { - g.mu.RLock() - conn, ok := g.connections[sessionID] - g.mu.RUnlock() - - if !ok { - return fmt.Errorf("connection not found for session: %s", sessionID) - } - - data, err := json.Marshal(message) - if err != nil { - return fmt.Errorf("failed to marshal message: %w", err) - } - - select { - case conn.send <- data: - return nil - case <-conn.ctx.Done(): - return fmt.Errorf("connection closed") - } -} - -// CloseConnection closes a connection -func (g *Gateway) CloseConnection(sessionID string) { - g.mu.Lock() - defer g.mu.Unlock() - - if conn, ok := g.connections[sessionID]; ok { - conn.cancel() - conn.ws.Close() - delete(g.connections, sessionID) - } -} - -// readPump reads messages from the WebSocket -func (c *Connection) readPump(gateway *Gateway) { - defer func() { - gateway.CloseConnection(c.sessionID) - c.ws.Close() - }() - - c.ws.SetReadDeadline(time.Now().Add(60 * time.Second)) - c.ws.SetPongHandler(func(string) error { - c.ws.SetReadDeadline(time.Now().Add(60 * time.Second)) - return nil - }) - - for { - _, message, err := c.ws.ReadMessage() - if err != nil { - if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { - log.Printf("WebSocket error: %v", err) - } - break - } - - // Handle incoming message (ICE candidates, SDP offers/answers, etc.) - var msg map[string]interface{} - if err := json.Unmarshal(message, &msg); err != nil { - log.Printf("Failed to unmarshal message: %v", err) - continue - } - - // Route message based on type - msgType, ok := msg["type"].(string) - if !ok { - continue - } - - switch msgType { - case "ice-candidate": - // Handle ICE candidate - case "offer": - // Handle SDP offer - case "answer": - // Handle SDP answer - default: - log.Printf("Unknown message type: %s", msgType) - } - } -} - -// writePump writes messages to the WebSocket -func (c *Connection) writePump() { - ticker := time.NewTicker(54 * time.Second) - defer func() { - ticker.Stop() - c.ws.Close() - }() - - for { - select { - case message, ok := <-c.send: - c.ws.SetWriteDeadline(time.Now().Add(10 * time.Second)) - if !ok { - c.ws.WriteMessage(websocket.CloseMessage, []byte{}) - return - } - - w, err := c.ws.NextWriter(websocket.TextMessage) - if err != nil { - return - } - w.Write(message) - - // Add queued messages - n := len(c.send) - for i := 0; i < n; i++ { - w.Write([]byte{'\n'}) - w.Write(<-c.send) - } - - if err := w.Close(); err != nil { - return - } - - case <-ticker.C: - c.ws.SetWriteDeadline(time.Now().Add(10 * time.Second)) - if err := c.ws.WriteMessage(websocket.PingMessage, nil); err != nil { - return - } - - case <-c.ctx.Done(): - return - } - } -} diff --git a/virtual-banker/backend/safety/filter.go b/virtual-banker/backend/safety/filter.go deleted file mode 100644 index d0a229e..0000000 --- a/virtual-banker/backend/safety/filter.go +++ /dev/null @@ -1,68 +0,0 @@ -package safety - -import ( - "context" - "strings" -) - -// Filter filters content for safety -type Filter interface { - Filter(ctx context.Context, text string) (*FilterResult, error) -} - -// FilterResult contains filtering results -type FilterResult struct { - Allowed bool - Blocked bool - Redacted string - Categories []string // e.g., "profanity", "pii", "abuse" -} - -// ContentFilter implements content filtering -type ContentFilter struct { - blockedWords []string -} - -// NewContentFilter creates a new content filter -func NewContentFilter() *ContentFilter { - return &ContentFilter{ - blockedWords: []string{ - // Add blocked words/phrases - }, - } -} - -// Filter filters content -func (f *ContentFilter) Filter(ctx context.Context, text string) (*FilterResult, error) { - lowerText := strings.ToLower(text) - var categories []string - - // Check for blocked words - for _, word := range f.blockedWords { - if strings.Contains(lowerText, strings.ToLower(word)) { - categories = append(categories, "profanity") - return &FilterResult{ - Allowed: false, - Blocked: true, - Redacted: f.redactPII(text), - Categories: categories, - }, nil - } - } - - // TODO: Add more sophisticated filtering (ML models, etc.) - - return &FilterResult{ - Allowed: true, - Blocked: false, - Redacted: f.redactPII(text), - }, nil -} - -// redactPII redacts personally identifiable information -func (f *ContentFilter) redactPII(text string) string { - // TODO: Implement PII detection and redaction - // For now, return as-is - return text -} - diff --git a/virtual-banker/backend/safety/rate_limit.go b/virtual-banker/backend/safety/rate_limit.go deleted file mode 100644 index daab616..0000000 --- a/virtual-banker/backend/safety/rate_limit.go +++ /dev/null @@ -1,59 +0,0 @@ -package safety - -import ( - "context" - "fmt" - "time" - - "github.com/redis/go-redis/v9" -) - -// RateLimiter implements rate limiting -type RateLimiter struct { - redis *redis.Client -} - -// NewRateLimiter creates a new rate limiter -func NewRateLimiter(redisClient *redis.Client) *RateLimiter { - return &RateLimiter{ - redis: redisClient, - } -} - -// Check checks if a request is within rate limits -func (r *RateLimiter) Check(ctx context.Context, key string, limit int, window time.Duration) (bool, error) { - // Use sliding window log algorithm - now := time.Now() - windowStart := now.Add(-window) - - // Count requests in window - count, err := r.redis.ZCount(ctx, key, fmt.Sprintf("%d", windowStart.Unix()), fmt.Sprintf("%d", now.Unix())).Result() - if err != nil { - return false, err - } - - if count >= int64(limit) { - return false, nil - } - - // Add current request - _, err = r.redis.ZAdd(ctx, key, redis.Z{ - Score: float64(now.Unix()), - Member: fmt.Sprintf("%d", now.UnixNano()), - }).Result() - if err != nil { - return false, err - } - - // Expire old entries - r.redis.Expire(ctx, key, window) - - return true, nil -} - -// CheckUser checks rate limit for a user -func (r *RateLimiter) CheckUser(ctx context.Context, tenantID, userID string, limit int, window time.Duration) (bool, error) { - key := fmt.Sprintf("ratelimit:user:%s:%s", tenantID, userID) - return r.Check(ctx, key, limit, window) -} - diff --git a/virtual-banker/backend/session/session.go b/virtual-banker/backend/session/session.go deleted file mode 100644 index 7be9186..0000000 --- a/virtual-banker/backend/session/session.go +++ /dev/null @@ -1,316 +0,0 @@ -package session - -import ( - "context" - "crypto/rand" - "encoding/base64" - "errors" - "fmt" - "time" - - "github.com/jackc/pgx/v5/pgxpool" - "github.com/redis/go-redis/v9" -) - -// Session represents a Virtual Banker session -type Session struct { - ID string - TenantID string - UserID string - EphemeralToken string - Config *TenantConfig - CreatedAt time.Time - ExpiresAt time.Time - LastActivityAt time.Time -} - -// TenantConfig holds tenant-specific configuration -type TenantConfig struct { - Theme map[string]interface{} `json:"theme"` - AvatarEnabled bool `json:"avatar_enabled"` - Greeting string `json:"greeting"` - AllowedTools []string `json:"allowed_tools"` - Policy *PolicyConfig `json:"policy"` -} - -// PolicyConfig holds policy settings -type PolicyConfig struct { - MaxSessionDuration time.Duration `json:"max_session_duration"` - RateLimitPerMinute int `json:"rate_limit_per_minute"` - RequireConsent bool `json:"require_consent"` -} - -// Manager manages sessions -type Manager struct { - db *pgxpool.Pool - redis *redis.Client -} - -// NewManager creates a new session manager -func NewManager(db *pgxpool.Pool, redisClient *redis.Client) *Manager { - return &Manager{ - db: db, - redis: redisClient, - } -} - -// CreateSession creates a new session -func (m *Manager) CreateSession(ctx context.Context, tenantID, userID string, authAssertion string) (*Session, error) { - // Validate JWT/auth assertion (simplified - should validate with tenant JWKs) - if authAssertion == "" { - return nil, errors.New("auth assertion required") - } - - // Load tenant config - config, err := m.loadTenantConfig(ctx, tenantID) - if err != nil { - return nil, fmt.Errorf("failed to load tenant config: %w", err) - } - - // Generate session ID - sessionID, err := generateSessionID() - if err != nil { - return nil, fmt.Errorf("failed to generate session ID: %w", err) - } - - // Generate ephemeral token - ephemeralToken, err := generateEphemeralToken() - if err != nil { - return nil, fmt.Errorf("failed to generate ephemeral token: %w", err) - } - - now := time.Now() - sessionDuration := config.Policy.MaxSessionDuration - if sessionDuration == 0 { - sessionDuration = 30 * time.Minute // default - } - - session := &Session{ - ID: sessionID, - TenantID: tenantID, - UserID: userID, - EphemeralToken: ephemeralToken, - Config: config, - CreatedAt: now, - ExpiresAt: now.Add(sessionDuration), - LastActivityAt: now, - } - - // Save to database - if err := m.saveSessionToDB(ctx, session); err != nil { - return nil, fmt.Errorf("failed to save session: %w", err) - } - - // Cache in Redis - if err := m.cacheSession(ctx, session); err != nil { - return nil, fmt.Errorf("failed to cache session: %w", err) - } - - return session, nil -} - -// GetSession retrieves a session by ID -func (m *Manager) GetSession(ctx context.Context, sessionID string) (*Session, error) { - // Try Redis first - session, err := m.getSessionFromCache(ctx, sessionID) - if err == nil && session != nil { - return session, nil - } - - // Fallback to database - session, err = m.getSessionFromDB(ctx, sessionID) - if err != nil { - return nil, fmt.Errorf("session not found: %w", err) - } - - // Cache it - _ = m.cacheSession(ctx, session) - - return session, nil -} - -// RefreshToken refreshes the ephemeral token for a session -func (m *Manager) RefreshToken(ctx context.Context, sessionID string) (string, error) { - session, err := m.GetSession(ctx, sessionID) - if err != nil { - return "", err - } - - // Check if session is expired - if time.Now().After(session.ExpiresAt) { - return "", errors.New("session expired") - } - - // Generate new token - newToken, err := generateEphemeralToken() - if err != nil { - return "", fmt.Errorf("failed to generate token: %w", err) - } - - session.EphemeralToken = newToken - session.LastActivityAt = time.Now() - - // Update in database and cache - if err := m.saveSessionToDB(ctx, session); err != nil { - return "", fmt.Errorf("failed to update session: %w", err) - } - _ = m.cacheSession(ctx, session) - - return newToken, nil -} - -// EndSession ends a session -func (m *Manager) EndSession(ctx context.Context, sessionID string) error { - // Remove from Redis - _ = m.redis.Del(ctx, fmt.Sprintf("session:%s", sessionID)) - - // Mark as ended in database - query := `UPDATE sessions SET ended_at = $1 WHERE id = $2` - _, err := m.db.Exec(ctx, query, time.Now(), sessionID) - return err -} - -// loadTenantConfig loads tenant configuration -func (m *Manager) loadTenantConfig(ctx context.Context, tenantID string) (*TenantConfig, error) { - query := ` - SELECT theme, avatar_enabled, greeting, allowed_tools, policy - FROM tenants - WHERE id = $1 - ` - - var config TenantConfig - var themeJSON, policyJSON []byte - var allowedToolsJSON []byte - - err := m.db.QueryRow(ctx, query, tenantID).Scan( - &themeJSON, - &config.AvatarEnabled, - &config.Greeting, - &allowedToolsJSON, - &policyJSON, - ) - if err != nil { - // Return default config if tenant not found - return &TenantConfig{ - Theme: map[string]interface{}{"primaryColor": "#0066cc"}, - AvatarEnabled: true, - Greeting: "Hello! How can I help you today?", - AllowedTools: []string{}, - Policy: &PolicyConfig{ - MaxSessionDuration: 30 * time.Minute, - RateLimitPerMinute: 10, - RequireConsent: true, - }, - }, nil - } - - // Parse JSON fields (simplified - should use json.Unmarshal) - // For now, return default with basic parsing - config.Policy = &PolicyConfig{ - MaxSessionDuration: 30 * time.Minute, - RateLimitPerMinute: 10, - RequireConsent: true, - } - - return &config, nil -} - -// saveSessionToDB saves session to database -func (m *Manager) saveSessionToDB(ctx context.Context, session *Session) error { - query := ` - INSERT INTO sessions (id, tenant_id, user_id, ephemeral_token, created_at, expires_at, last_activity_at) - VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (id) DO UPDATE SET - ephemeral_token = $4, - last_activity_at = $7 - ` - - _, err := m.db.Exec(ctx, query, - session.ID, - session.TenantID, - session.UserID, - session.EphemeralToken, - session.CreatedAt, - session.ExpiresAt, - session.LastActivityAt, - ) - return err -} - -// getSessionFromDB retrieves session from database -func (m *Manager) getSessionFromDB(ctx context.Context, sessionID string) (*Session, error) { - query := ` - SELECT id, tenant_id, user_id, ephemeral_token, created_at, expires_at, last_activity_at - FROM sessions - WHERE id = $1 AND ended_at IS NULL - ` - - var session Session - err := m.db.QueryRow(ctx, query, sessionID).Scan( - &session.ID, - &session.TenantID, - &session.UserID, - &session.EphemeralToken, - &session.CreatedAt, - &session.ExpiresAt, - &session.LastActivityAt, - ) - if err != nil { - return nil, err - } - - // Load config - config, err := m.loadTenantConfig(ctx, session.TenantID) - if err != nil { - return nil, err - } - session.Config = config - - return &session, nil -} - -// cacheSession caches session in Redis -func (m *Manager) cacheSession(ctx context.Context, session *Session) error { - key := fmt.Sprintf("session:%s", session.ID) - ttl := time.Until(session.ExpiresAt) - if ttl <= 0 { - return nil - } - - // Store as JSON (simplified - should serialize properly) - return m.redis.Set(ctx, key, session.ID, ttl).Err() -} - -// getSessionFromCache retrieves session from Redis cache -func (m *Manager) getSessionFromCache(ctx context.Context, sessionID string) (*Session, error) { - key := fmt.Sprintf("session:%s", sessionID) - val, err := m.redis.Get(ctx, key).Result() - if err != nil { - return nil, err - } - - if val != sessionID { - return nil, errors.New("cache mismatch") - } - - // If cached, fetch full session from DB - return m.getSessionFromDB(ctx, sessionID) -} - -// generateSessionID generates a unique session ID -func generateSessionID() (string, error) { - b := make([]byte, 16) - if _, err := rand.Read(b); err != nil { - return "", err - } - return base64.URLEncoding.EncodeToString(b), nil -} - -// generateEphemeralToken generates an ephemeral token -func generateEphemeralToken() (string, error) { - b := make([]byte, 32) - if _, err := rand.Read(b); err != nil { - return "", err - } - return base64.URLEncoding.EncodeToString(b), nil -} diff --git a/virtual-banker/backend/tools/banking/account_status.go b/virtual-banker/backend/tools/banking/account_status.go deleted file mode 100644 index f9c43ec..0000000 --- a/virtual-banker/backend/tools/banking/account_status.go +++ /dev/null @@ -1,68 +0,0 @@ -package banking - -import ( - "context" - - "github.com/explorer/virtual-banker/backend/tools" -) - -// AccountStatusTool gets account status -type AccountStatusTool struct { - client *BankingClient -} - -// NewAccountStatusTool creates a new account status tool -func NewAccountStatusTool() *AccountStatusTool { - return &AccountStatusTool{ - client: NewBankingClient(getBankingAPIURL()), - } -} - -// getBankingAPIURL gets the banking API URL from environment -func getBankingAPIURL() string { - // Default to main API URL - return "http://localhost:8080" -} - -// Name returns the tool name -func (t *AccountStatusTool) Name() string { - return "get_account_status" -} - -// Description returns the tool description -func (t *AccountStatusTool) Description() string { - return "Get the status of a bank account including balance, transactions, and account details" -} - -// Execute executes the tool -func (t *AccountStatusTool) Execute(ctx context.Context, params map[string]interface{}) (*tools.ToolResult, error) { - accountID, ok := params["account_id"].(string) - if !ok || accountID == "" { - return &tools.ToolResult{ - Success: false, - Error: "account_id is required", - }, nil - } - - // Call banking service - data, err := t.client.GetAccountStatus(ctx, accountID) - if err != nil { - // Fallback to mock data if service unavailable - return &tools.ToolResult{ - Success: true, - Data: map[string]interface{}{ - "account_id": accountID, - "balance": 10000.00, - "currency": "USD", - "status": "active", - "type": "checking", - "note": "Using fallback data - banking service unavailable", - }, - }, nil - } - - return &tools.ToolResult{ - Success: true, - Data: data, - }, nil -} diff --git a/virtual-banker/backend/tools/banking/create_ticket.go b/virtual-banker/backend/tools/banking/create_ticket.go deleted file mode 100644 index d9d09d8..0000000 --- a/virtual-banker/backend/tools/banking/create_ticket.go +++ /dev/null @@ -1,66 +0,0 @@ -package banking - -import ( - "context" - "fmt" - - "github.com/explorer/virtual-banker/backend/tools" -) - -// CreateTicketTool creates a support ticket -type CreateTicketTool struct { - client *BankingClient -} - -// NewCreateTicketTool creates a new create ticket tool -func NewCreateTicketTool() *CreateTicketTool { - return &CreateTicketTool{ - client: NewBankingClient(getBankingAPIURL()), - } -} - -// Name returns the tool name -func (t *CreateTicketTool) Name() string { - return "create_support_ticket" -} - -// Description returns the tool description -func (t *CreateTicketTool) Description() string { - return "Create a support ticket for customer service" -} - -// Execute executes the tool -func (t *CreateTicketTool) Execute(ctx context.Context, params map[string]interface{}) (*tools.ToolResult, error) { - subject, _ := params["subject"].(string) - details, _ := params["details"].(string) - - if subject == "" { - return &tools.ToolResult{ - Success: false, - Error: "subject is required", - }, nil - } - - // Call banking service - data, err := t.client.CreateTicket(ctx, subject, details) - if err != nil { - // Fallback to mock data if service unavailable - return &tools.ToolResult{ - Success: true, - Data: map[string]interface{}{ - "ticket_id": fmt.Sprintf("TKT-%d", 12345), - "subject": subject, - "status": "open", - "note": "Using fallback data - banking service unavailable", - }, - RequiresConfirmation: false, - }, nil - } - - return &tools.ToolResult{ - Success: true, - Data: data, - RequiresConfirmation: false, - }, nil -} - diff --git a/virtual-banker/backend/tools/banking/integration.go b/virtual-banker/backend/tools/banking/integration.go deleted file mode 100644 index 2a59f03..0000000 --- a/virtual-banker/backend/tools/banking/integration.go +++ /dev/null @@ -1,91 +0,0 @@ -package banking - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "time" -) - -// BankingClient provides access to backend banking services -type BankingClient struct { - baseURL string - httpClient *http.Client -} - -// NewBankingClient creates a new banking client -func NewBankingClient(baseURL string) *BankingClient { - return &BankingClient{ - baseURL: baseURL, - httpClient: &http.Client{ - Timeout: 10 * time.Second, - }, - } -} - -// GetAccountStatus gets account status from banking service -func (c *BankingClient) GetAccountStatus(ctx context.Context, accountID string) (map[string]interface{}, error) { - url := fmt.Sprintf("%s/api/v1/banking/accounts/%s", c.baseURL, accountID) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status: %d", resp.StatusCode) - } - - var result map[string]interface{} - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, err - } - - return result, nil -} - -// CreateTicket creates a support ticket -func (c *BankingClient) CreateTicket(ctx context.Context, subject, details string) (map[string]interface{}, error) { - url := fmt.Sprintf("%s/api/v1/banking/tickets", c.baseURL) - - payload := map[string]string{ - "subject": subject, - "details": details, - } - - jsonData, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("unexpected status: %d", resp.StatusCode) - } - - var result map[string]interface{} - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, err - } - - return result, nil -} diff --git a/virtual-banker/backend/tools/banking/payment.go b/virtual-banker/backend/tools/banking/payment.go deleted file mode 100644 index 5dff6a5..0000000 --- a/virtual-banker/backend/tools/banking/payment.go +++ /dev/null @@ -1,60 +0,0 @@ -package banking - -import ( - "context" - - "github.com/explorer/virtual-banker/backend/tools" -) - -// SubmitPaymentTool submits a payment -type SubmitPaymentTool struct{} - -// NewSubmitPaymentTool creates a new submit payment tool -func NewSubmitPaymentTool() *SubmitPaymentTool { - return &SubmitPaymentTool{} -} - -// Name returns the tool name -func (t *SubmitPaymentTool) Name() string { - return "submit_payment" -} - -// Description returns the tool description -func (t *SubmitPaymentTool) Description() string { - return "Submit a payment transaction (requires confirmation)" -} - -// Execute executes the tool -func (t *SubmitPaymentTool) Execute(ctx context.Context, params map[string]interface{}) (*tools.ToolResult, error) { - amount, _ := params["amount"].(float64) - method, _ := params["method"].(string) - - if amount <= 0 { - return &tools.ToolResult{ - Success: false, - Error: "amount must be greater than 0", - }, nil - } - - if method == "" { - return &tools.ToolResult{ - Success: false, - Error: "payment method is required", - }, nil - } - - // TODO: Call backend/banking/payments/ service - // For now, return mock data - return &tools.ToolResult{ - Success: true, - Data: map[string]interface{}{ - "payment_id": "PAY-11111", - "amount": amount, - "method": method, - "status": "pending_confirmation", - "transaction_id": "TXN-22222", - }, - RequiresConfirmation: true, // Payments always require confirmation - }, nil -} - diff --git a/virtual-banker/backend/tools/banking/schedule.go b/virtual-banker/backend/tools/banking/schedule.go deleted file mode 100644 index 5b068d3..0000000 --- a/virtual-banker/backend/tools/banking/schedule.go +++ /dev/null @@ -1,62 +0,0 @@ -package banking - -import ( - "context" - "time" - - "github.com/explorer/virtual-banker/backend/tools" -) - -// ScheduleAppointmentTool schedules an appointment -type ScheduleAppointmentTool struct{} - -// NewScheduleAppointmentTool creates a new schedule appointment tool -func NewScheduleAppointmentTool() *ScheduleAppointmentTool { - return &ScheduleAppointmentTool{} -} - -// Name returns the tool name -func (t *ScheduleAppointmentTool) Name() string { - return "schedule_appointment" -} - -// Description returns the tool description -func (t *ScheduleAppointmentTool) Description() string { - return "Schedule an appointment with a bank representative" -} - -// Execute executes the tool -func (t *ScheduleAppointmentTool) Execute(ctx context.Context, params map[string]interface{}) (*tools.ToolResult, error) { - datetime, _ := params["datetime"].(string) - reason, _ := params["reason"].(string) - - if datetime == "" { - return &tools.ToolResult{ - Success: false, - Error: "datetime is required", - }, nil - } - - // Parse datetime - _, err := time.Parse(time.RFC3339, datetime) - if err != nil { - return &tools.ToolResult{ - Success: false, - Error: "invalid datetime format (use RFC3339)", - }, nil - } - - // TODO: Call backend/banking/ service to schedule appointment - // For now, return mock data - return &tools.ToolResult{ - Success: true, - Data: map[string]interface{}{ - "appointment_id": "APT-67890", - "datetime": datetime, - "reason": reason, - "status": "scheduled", - }, - RequiresConfirmation: true, // Appointments require confirmation - }, nil -} - diff --git a/virtual-banker/backend/tools/executor.go b/virtual-banker/backend/tools/executor.go deleted file mode 100644 index 3130490..0000000 --- a/virtual-banker/backend/tools/executor.go +++ /dev/null @@ -1,89 +0,0 @@ -package tools - -import ( - "context" - "fmt" -) - -// Executor executes tools -type Executor struct { - registry *Registry - auditLog AuditLogger -} - -// NewExecutor creates a new tool executor -func NewExecutor(registry *Registry, auditLog AuditLogger) *Executor { - return &Executor{ - registry: registry, - auditLog: auditLog, - } -} - -// Execute executes a tool -func (e *Executor) Execute(ctx context.Context, toolName string, params map[string]interface{}, userID, tenantID string) (*ToolResult, error) { - tool, err := e.registry.Get(toolName) - if err != nil { - return nil, err - } - - // Log execution attempt - e.auditLog.LogToolExecution(ctx, &ToolExecutionLog{ - ToolName: toolName, - UserID: userID, - TenantID: tenantID, - Params: params, - Status: "executing", - }) - - // Execute tool - result, err := tool.Execute(ctx, params) - if err != nil { - e.auditLog.LogToolExecution(ctx, &ToolExecutionLog{ - ToolName: toolName, - UserID: userID, - TenantID: tenantID, - Params: params, - Status: "failed", - Error: err.Error(), - }) - return nil, err - } - - // Log result - e.auditLog.LogToolExecution(ctx, &ToolExecutionLog{ - ToolName: toolName, - UserID: userID, - TenantID: tenantID, - Params: params, - Status: "completed", - Result: result.Data, - }) - - return result, nil -} - -// AuditLogger logs tool executions -type AuditLogger interface { - LogToolExecution(ctx context.Context, log *ToolExecutionLog) -} - -// ToolExecutionLog represents a tool execution log entry -type ToolExecutionLog struct { - ToolName string - UserID string - TenantID string - Params map[string]interface{} - Status string - Error string - Result interface{} -} - -// MockAuditLogger is a mock audit logger -type MockAuditLogger struct{} - -// LogToolExecution logs a tool execution -func (m *MockAuditLogger) LogToolExecution(ctx context.Context, log *ToolExecutionLog) { - // Mock implementation - in production, write to database - fmt.Printf("Tool execution: %s by %s (%s) - %s\n", log.ToolName, log.UserID, log.TenantID, log.Status) -} - diff --git a/virtual-banker/backend/tools/registry.go b/virtual-banker/backend/tools/registry.go deleted file mode 100644 index d3dbcdd..0000000 --- a/virtual-banker/backend/tools/registry.go +++ /dev/null @@ -1,73 +0,0 @@ -package tools - -import ( - "context" - "fmt" -) - -// Tool represents an executable tool -type Tool interface { - Name() string - Description() string - Execute(ctx context.Context, params map[string]interface{}) (*ToolResult, error) -} - -// ToolResult represents the result of tool execution -type ToolResult struct { - Success bool - Data interface{} - Error string - RequiresConfirmation bool -} - -// Registry manages available tools -type Registry struct { - tools map[string]Tool -} - -// NewRegistry creates a new tool registry -func NewRegistry() *Registry { - return &Registry{ - tools: make(map[string]Tool), - } -} - -// Register registers a tool -func (r *Registry) Register(tool Tool) { - r.tools[tool.Name()] = tool -} - -// Get gets a tool by name -func (r *Registry) Get(name string) (Tool, error) { - tool, ok := r.tools[name] - if !ok { - return nil, fmt.Errorf("tool not found: %s", name) - } - return tool, nil -} - -// List returns all registered tools -func (r *Registry) List() []Tool { - tools := make([]Tool, 0, len(r.tools)) - for _, tool := range r.tools { - tools = append(tools, tool) - } - return tools -} - -// GetAllowedTools returns tools allowed for a tenant -func (r *Registry) GetAllowedTools(allowedNames []string) []Tool { - allowedSet := make(map[string]bool) - for _, name := range allowedNames { - allowedSet[name] = true - } - - var tools []Tool - for _, tool := range r.tools { - if allowedSet[tool.Name()] { - tools = append(tools, tool) - } - } - return tools -} - diff --git a/virtual-banker/backend/tts/elevenlabs-adapter.go b/virtual-banker/backend/tts/elevenlabs-adapter.go deleted file mode 100644 index 2fb2948..0000000 --- a/virtual-banker/backend/tts/elevenlabs-adapter.go +++ /dev/null @@ -1,329 +0,0 @@ -package tts - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" -) - -// ElevenLabsTTSService integrates with ElevenLabs TTS API -type ElevenLabsTTSService struct { - apiKey string - voiceID string - modelID string - baseURL string - httpClient *http.Client - defaultVoiceConfig *VoiceConfig -} - -// VoiceConfig holds ElevenLabs voice configuration -type VoiceConfig struct { - Stability float64 `json:"stability"` - SimilarityBoost float64 `json:"similarity_boost"` - Style float64 `json:"style,omitempty"` - UseSpeakerBoost bool `json:"use_speaker_boost,omitempty"` -} - -// ElevenLabsRequest represents the request body for ElevenLabs API -type ElevenLabsRequest struct { - Text string `json:"text"` - ModelID string `json:"model_id,omitempty"` - VoiceSettings VoiceConfig `json:"voice_settings,omitempty"` -} - -// NewElevenLabsTTSService creates a new ElevenLabs TTS service -func NewElevenLabsTTSService(apiKey, voiceID string) *ElevenLabsTTSService { - return &ElevenLabsTTSService{ - apiKey: apiKey, - voiceID: voiceID, - modelID: "eleven_multilingual_v2", // Default model - baseURL: "https://api.elevenlabs.io/v1", - httpClient: &http.Client{ - Timeout: 30 * time.Second, - }, - defaultVoiceConfig: &VoiceConfig{ - Stability: 0.5, - SimilarityBoost: 0.75, - UseSpeakerBoost: true, - }, - } -} - -// SetModelID sets the model ID for synthesis -func (s *ElevenLabsTTSService) SetModelID(modelID string) { - s.modelID = modelID -} - -// SetVoiceConfig sets the default voice configuration -func (s *ElevenLabsTTSService) SetVoiceConfig(config *VoiceConfig) { - s.defaultVoiceConfig = config -} - -// Synthesize synthesizes text to audio using ElevenLabs REST API -func (s *ElevenLabsTTSService) Synthesize(ctx context.Context, text string) ([]byte, error) { - return s.SynthesizeWithConfig(ctx, text, s.defaultVoiceConfig) -} - -// SynthesizeWithConfig synthesizes text to audio with custom voice configuration -func (s *ElevenLabsTTSService) SynthesizeWithConfig(ctx context.Context, text string, config *VoiceConfig) ([]byte, error) { - if s.apiKey == "" { - return nil, fmt.Errorf("ElevenLabs API key not configured") - } - if s.voiceID == "" { - return nil, fmt.Errorf("ElevenLabs voice ID not configured") - } - if text == "" { - return nil, fmt.Errorf("text cannot be empty") - } - - // Use default config if none provided - if config == nil { - config = s.defaultVoiceConfig - } - - // Prepare request body - reqBody := ElevenLabsRequest{ - Text: text, - ModelID: s.modelID, - VoiceSettings: *config, - } - - jsonBody, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) - } - - // Build request URL - url := fmt.Sprintf("%s/text-to-speech/%s", s.baseURL, s.voiceID) - - // Create HTTP request - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonBody)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Accept", "audio/mpeg") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("xi-api-key", s.apiKey) - - // Execute request with retry logic - var resp *http.Response - maxRetries := 3 - for i := 0; i < maxRetries; i++ { - resp, err = s.httpClient.Do(req) - if err == nil && resp.StatusCode == http.StatusOK { - break - } - - if err != nil { - if i < maxRetries-1 { - // Exponential backoff - backoff := time.Duration(i+1) * time.Second - time.Sleep(backoff) - continue - } - return nil, fmt.Errorf("failed to call ElevenLabs API after %d retries: %w", maxRetries, err) - } - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - bodyBytes, _ := io.ReadAll(bytes.NewReader([]byte{})) - if resp.Body != nil { - bodyBytes, _ = io.ReadAll(resp.Body) - } - - // Retry on 5xx errors - if resp.StatusCode >= 500 && i < maxRetries-1 { - backoff := time.Duration(i+1) * time.Second - time.Sleep(backoff) - continue - } - - return nil, fmt.Errorf("ElevenLabs API error: status %d, body: %s", resp.StatusCode, string(bodyBytes)) - } - } - defer resp.Body.Close() - - // Read audio data - audioData, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read audio data: %w", err) - } - - return audioData, nil -} - -// SynthesizeStream synthesizes text to audio using ElevenLabs streaming API -func (s *ElevenLabsTTSService) SynthesizeStream(ctx context.Context, text string) (io.Reader, error) { - return s.SynthesizeStreamWithConfig(ctx, text, s.defaultVoiceConfig) -} - -// SynthesizeStreamWithConfig synthesizes text to audio stream with custom voice configuration -func (s *ElevenLabsTTSService) SynthesizeStreamWithConfig(ctx context.Context, text string, config *VoiceConfig) (io.Reader, error) { - if s.apiKey == "" { - return nil, fmt.Errorf("ElevenLabs API key not configured") - } - if s.voiceID == "" { - return nil, fmt.Errorf("ElevenLabs voice ID not configured") - } - if text == "" { - return nil, fmt.Errorf("text cannot be empty") - } - - // Use default config if none provided - if config == nil { - config = s.defaultVoiceConfig - } - - // Prepare request body - reqBody := ElevenLabsRequest{ - Text: text, - ModelID: s.modelID, - VoiceSettings: *config, - } - - jsonBody, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) - } - - // Build request URL for streaming - url := fmt.Sprintf("%s/text-to-speech/%s/stream", s.baseURL, s.voiceID) - - // Create HTTP request - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonBody)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Accept", "audio/mpeg") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("xi-api-key", s.apiKey) - - // Execute request - resp, err := s.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to call ElevenLabs streaming API: %w", err) - } - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - bodyBytes, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("ElevenLabs streaming API error: status %d, body: %s", resp.StatusCode, string(bodyBytes)) - } - - // Return stream reader (caller is responsible for closing) - return resp.Body, nil -} - -// GetVisemes returns viseme events for lip sync -// ElevenLabs doesn't provide viseme data directly, so we use phoneme-to-viseme mapping -func (s *ElevenLabsTTSService) GetVisemes(ctx context.Context, text string) ([]VisemeEvent, error) { - if text == "" { - return nil, fmt.Errorf("text cannot be empty") - } - - // Use phoneme-to-viseme mapping to generate viseme events - // This is a simplified implementation - in production, you might want to use - // a more sophisticated phoneme-to-viseme mapping service or library - visemes := s.generateVisemesFromText(text) - - return visemes, nil -} - -// generateVisemesFromText generates viseme events from text using basic phoneme-to-viseme mapping -// This is a simplified implementation. For production, consider using: -// - A dedicated phoneme-to-viseme mapping service -// - A TTS provider that provides phoneme timing data (e.g., Azure TTS with SSML) -// - Integration with a speech analysis library -func (s *ElevenLabsTTSService) generateVisemesFromText(text string) []VisemeEvent { - // Basic phoneme-to-viseme mapping - phonemeToViseme := map[string]string{ - // Vowels - "aa": "aa", "ae": "aa", "ah": "aa", "ao": "aa", "aw": "aa", - "ay": "aa", "eh": "ee", "er": "er", "ey": "ee", "ih": "ee", - "iy": "ee", "ow": "oh", "oy": "oh", "uh": "ou", "uw": "ou", - // Consonants - "b": "aa", "p": "aa", "m": "aa", - "f": "ee", "v": "ee", - "th": "ee", - "d": "aa", "t": "aa", "n": "aa", "l": "aa", - "k": "aa", "g": "aa", "ng": "aa", - "s": "ee", "z": "ee", - "sh": "ee", "zh": "ee", "ch": "ee", "jh": "ee", - "y": "ee", - "w": "ou", - "r": "er", - "h": "sil", - "sil": "sil", "sp": "sil", - } - - // Simple word-to-phoneme approximation - // In production, use a proper TTS API that provides phoneme timing or a phoneme-to-viseme service - words := strings.Fields(strings.ToLower(text)) - visemes := []VisemeEvent{} - currentTime := 0.0 - durationPerWord := 0.3 // Approximate duration per word in seconds - initialPause := 0.1 - - // Initial silence - visemes = append(visemes, VisemeEvent{ - Viseme: "sil", - StartTime: currentTime, - EndTime: currentTime + initialPause, - Phoneme: "sil", - }) - currentTime += initialPause - - // Generate visemes for each word - for _, word := range words { - // Simple approximation: map first phoneme to viseme - viseme := "aa" // default - if len(word) > 0 { - firstChar := string(word[0]) - if mapped, ok := phonemeToViseme[firstChar]; ok { - viseme = mapped - } else { - // Map common starting consonants - switch firstChar { - case "a", "e", "i", "o", "u": - viseme = "aa" - default: - viseme = "aa" - } - } - } - - visemes = append(visemes, VisemeEvent{ - Viseme: viseme, - StartTime: currentTime, - EndTime: currentTime + durationPerWord, - Phoneme: word, - }) - currentTime += durationPerWord - - // Small pause between words - visemes = append(visemes, VisemeEvent{ - Viseme: "sil", - StartTime: currentTime, - EndTime: currentTime + 0.05, - Phoneme: "sil", - }) - currentTime += 0.05 - } - - // Final silence - visemes = append(visemes, VisemeEvent{ - Viseme: "sil", - StartTime: currentTime, - EndTime: currentTime + 0.1, - Phoneme: "sil", - }) - - return visemes -} diff --git a/virtual-banker/backend/tts/service.go b/virtual-banker/backend/tts/service.go deleted file mode 100644 index 7e34a97..0000000 --- a/virtual-banker/backend/tts/service.go +++ /dev/null @@ -1,58 +0,0 @@ -package tts - -import ( - "context" - "fmt" - "io" -) - -// Service provides text-to-speech functionality -type Service interface { - SynthesizeStream(ctx context.Context, text string) (io.Reader, error) - Synthesize(ctx context.Context, text string) ([]byte, error) - GetVisemes(ctx context.Context, text string) ([]VisemeEvent, error) -} - -// VisemeEvent represents a viseme (lip shape) event for lip sync -type VisemeEvent struct { - Viseme string `json:"viseme"` // e.g., "sil", "aa", "ee", "oh", "ou" - StartTime float64 `json:"start_time"` - EndTime float64 `json:"end_time"` - Phoneme string `json:"phoneme,omitempty"` -} - -// MockTTSService is a mock implementation for development -type MockTTSService struct{} - -// NewMockTTSService creates a new mock TTS service -func NewMockTTSService() *MockTTSService { - return &MockTTSService{} -} - -// SynthesizeStream synthesizes text to audio stream -func (s *MockTTSService) SynthesizeStream(ctx context.Context, text string) (io.Reader, error) { - // Mock implementation - in production, integrate with ElevenLabs, Azure TTS, etc. - // For now, return empty reader - return io.NopCloser(io.Reader(nil)), nil -} - -// Synthesize synthesizes text to audio -func (s *MockTTSService) Synthesize(ctx context.Context, text string) ([]byte, error) { - // Mock implementation - return []byte{}, nil -} - -// GetVisemes returns viseme events for lip sync -func (s *MockTTSService) GetVisemes(ctx context.Context, text string) ([]VisemeEvent, error) { - // Mock implementation - return basic visemes - return []VisemeEvent{ - {Viseme: "sil", StartTime: 0.0, EndTime: 0.1}, - {Viseme: "aa", StartTime: 0.1, EndTime: 0.3}, - {Viseme: "ee", StartTime: 0.3, EndTime: 0.5}, - }, nil -} - -// ElevenLabsTTSService integrates with ElevenLabs (implementation in elevenlabs-adapter.go) -// This interface definition is kept for backwards compatibility -// The actual implementation is in elevenlabs-adapter.go - diff --git a/virtual-banker/database/migrations/001_sessions.up.sql b/virtual-banker/database/migrations/001_sessions.up.sql deleted file mode 100644 index f05cad2..0000000 --- a/virtual-banker/database/migrations/001_sessions.up.sql +++ /dev/null @@ -1,55 +0,0 @@ --- Create sessions table -CREATE TABLE IF NOT EXISTS sessions ( - id VARCHAR(255) PRIMARY KEY, - tenant_id VARCHAR(255) NOT NULL, - user_id VARCHAR(255) NOT NULL, - ephemeral_token VARCHAR(512) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - expires_at TIMESTAMP NOT NULL, - last_activity_at TIMESTAMP NOT NULL DEFAULT NOW(), - ended_at TIMESTAMP, - INDEX idx_tenant_user (tenant_id, user_id), - INDEX idx_expires_at (expires_at), - INDEX idx_ended_at (ended_at) -); - --- Create tenants table -CREATE TABLE IF NOT EXISTS tenants ( - id VARCHAR(255) PRIMARY KEY, - name VARCHAR(255) NOT NULL, - theme JSONB, - avatar_enabled BOOLEAN DEFAULT true, - greeting TEXT, - allowed_tools JSONB DEFAULT '[]'::jsonb, - policy JSONB, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW() -); - --- Create conversations table -CREATE TABLE IF NOT EXISTS conversations ( - id VARCHAR(255) PRIMARY KEY, - session_id VARCHAR(255) NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - user_id VARCHAR(255) NOT NULL, - tenant_id VARCHAR(255) NOT NULL, - started_at TIMESTAMP NOT NULL DEFAULT NOW(), - ended_at TIMESTAMP, - metadata JSONB, - INDEX idx_session (session_id), - INDEX idx_user (user_id), - INDEX idx_tenant (tenant_id) -); - --- Create conversation_messages table -CREATE TABLE IF NOT EXISTS conversation_messages ( - id VARCHAR(255) PRIMARY KEY, - conversation_id VARCHAR(255) NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, - role VARCHAR(50) NOT NULL, -- 'user' or 'assistant' - content TEXT NOT NULL, - audio_url TEXT, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - metadata JSONB, - INDEX idx_conversation (conversation_id), - INDEX idx_created_at (created_at) -); - diff --git a/virtual-banker/database/migrations/002_conversations.up.sql b/virtual-banker/database/migrations/002_conversations.up.sql deleted file mode 100644 index e1c5108..0000000 --- a/virtual-banker/database/migrations/002_conversations.up.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Additional indexes for conversations -CREATE INDEX IF NOT EXISTS idx_conversations_started_at ON conversations(started_at); -CREATE INDEX IF NOT EXISTS idx_conversations_ended_at ON conversations(ended_at); - --- Create conversation_state table for workflow state -CREATE TABLE IF NOT EXISTS conversation_state ( - session_id VARCHAR(255) PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, - workflow VARCHAR(255), - step VARCHAR(255), - context JSONB DEFAULT '{}'::jsonb, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW(), - expires_at TIMESTAMP -); - diff --git a/virtual-banker/database/migrations/003_tenants.up.sql b/virtual-banker/database/migrations/003_tenants.up.sql deleted file mode 100644 index d43123b..0000000 --- a/virtual-banker/database/migrations/003_tenants.up.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Add default tenant if not exists -INSERT INTO tenants (id, name, theme, avatar_enabled, greeting, allowed_tools, policy) -VALUES ( - 'default', - 'Default Tenant', - '{"primaryColor": "#0066cc", "secondaryColor": "#004499"}'::jsonb, - true, - 'Hello! How can I help you today?', - '[]'::jsonb, - '{"max_session_duration_minutes": 30, "rate_limit_per_minute": 10, "require_consent": true}'::jsonb -) ON CONFLICT (id) DO NOTHING; - diff --git a/virtual-banker/database/migrations/004_vector_extension.up.sql b/virtual-banker/database/migrations/004_vector_extension.up.sql deleted file mode 100644 index 8cecfcf..0000000 --- a/virtual-banker/database/migrations/004_vector_extension.up.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Enable pgvector extension for RAG functionality -CREATE EXTENSION IF NOT EXISTS vector; - --- Create knowledge_base table for RAG -CREATE TABLE IF NOT EXISTS knowledge_base ( - id VARCHAR(255) PRIMARY KEY, - tenant_id VARCHAR(255) NOT NULL, - title VARCHAR(500), - content TEXT NOT NULL, - embedding vector(1536), -- OpenAI ada-002 dimension, adjust as needed - metadata JSONB, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW(), - INDEX idx_tenant (tenant_id) -); - --- Create index for vector similarity search -CREATE INDEX IF NOT EXISTS idx_knowledge_embedding ON knowledge_base -USING ivfflat (embedding vector_cosine_ops) -WITH (lists = 100); - diff --git a/virtual-banker/database/migrations/005_user_profiles.up.sql b/virtual-banker/database/migrations/005_user_profiles.up.sql deleted file mode 100644 index 06912bc..0000000 --- a/virtual-banker/database/migrations/005_user_profiles.up.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Create user_profiles table for memory service -CREATE TABLE IF NOT EXISTS user_profiles ( - user_id VARCHAR(255) NOT NULL, - tenant_id VARCHAR(255) NOT NULL, - preferences JSONB DEFAULT '{}'::jsonb, - context JSONB DEFAULT '{}'::jsonb, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW(), - PRIMARY KEY (user_id, tenant_id), - INDEX idx_tenant (tenant_id) -); - --- Create conversation_history table -CREATE TABLE IF NOT EXISTS conversation_history ( - id VARCHAR(255) PRIMARY KEY, - user_id VARCHAR(255) NOT NULL, - tenant_id VARCHAR(255) NOT NULL, - session_id VARCHAR(255) NOT NULL, - messages JSONB NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - INDEX idx_user_tenant (user_id, tenant_id), - INDEX idx_session (session_id), - INDEX idx_created_at (created_at) -); - diff --git a/virtual-banker/database/schema.sql b/virtual-banker/database/schema.sql deleted file mode 100644 index eced71a..0000000 --- a/virtual-banker/database/schema.sql +++ /dev/null @@ -1,82 +0,0 @@ --- Virtual Banker Database Schema --- This file contains the complete schema for reference - --- Sessions -CREATE TABLE IF NOT EXISTS sessions ( - id VARCHAR(255) PRIMARY KEY, - tenant_id VARCHAR(255) NOT NULL, - user_id VARCHAR(255) NOT NULL, - ephemeral_token VARCHAR(512) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - expires_at TIMESTAMP NOT NULL, - last_activity_at TIMESTAMP NOT NULL DEFAULT NOW(), - ended_at TIMESTAMP, - INDEX idx_tenant_user (tenant_id, user_id), - INDEX idx_expires_at (expires_at), - INDEX idx_ended_at (ended_at) -); - --- Tenants -CREATE TABLE IF NOT EXISTS tenants ( - id VARCHAR(255) PRIMARY KEY, - name VARCHAR(255) NOT NULL, - theme JSONB, - avatar_enabled BOOLEAN DEFAULT true, - greeting TEXT, - allowed_tools JSONB DEFAULT '[]'::jsonb, - policy JSONB, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW() -); - --- Conversations -CREATE TABLE IF NOT EXISTS conversations ( - id VARCHAR(255) PRIMARY KEY, - session_id VARCHAR(255) NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - user_id VARCHAR(255) NOT NULL, - tenant_id VARCHAR(255) NOT NULL, - started_at TIMESTAMP NOT NULL DEFAULT NOW(), - ended_at TIMESTAMP, - metadata JSONB, - INDEX idx_session (session_id), - INDEX idx_user (user_id), - INDEX idx_tenant (tenant_id) -); - --- Conversation Messages -CREATE TABLE IF NOT EXISTS conversation_messages ( - id VARCHAR(255) PRIMARY KEY, - conversation_id VARCHAR(255) NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, - role VARCHAR(50) NOT NULL, - content TEXT NOT NULL, - audio_url TEXT, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - metadata JSONB, - INDEX idx_conversation (conversation_id), - INDEX idx_created_at (created_at) -); - --- Conversation State -CREATE TABLE IF NOT EXISTS conversation_state ( - session_id VARCHAR(255) PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, - workflow VARCHAR(255), - step VARCHAR(255), - context JSONB DEFAULT '{}'::jsonb, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW(), - expires_at TIMESTAMP -); - --- Knowledge Base (requires pgvector extension) -CREATE TABLE IF NOT EXISTS knowledge_base ( - id VARCHAR(255) PRIMARY KEY, - tenant_id VARCHAR(255) NOT NULL, - title VARCHAR(500), - content TEXT NOT NULL, - embedding vector(1536), - metadata JSONB, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW(), - INDEX idx_tenant (tenant_id) -); - diff --git a/virtual-banker/deployment/Dockerfile.backend b/virtual-banker/deployment/Dockerfile.backend deleted file mode 100644 index 1c27f2c..0000000 --- a/virtual-banker/deployment/Dockerfile.backend +++ /dev/null @@ -1,30 +0,0 @@ -FROM golang:1.21-alpine AS builder - -WORKDIR /app - -# Copy go mod files -COPY go.mod go.sum ./ -RUN go mod download - -# Copy source code -COPY . . - -# Build -RUN CGO_ENABLED=0 GOOS=linux go build -o /app/virtual-banker-api ./main.go - -# Final stage -FROM alpine:latest - -RUN apk --no-cache add ca-certificates curl - -WORKDIR /root/ - -COPY --from=builder /app/virtual-banker-api . - -EXPOSE 8081 - -HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ - CMD curl -f http://localhost:8081/health || exit 1 - -CMD ["./virtual-banker-api"] - diff --git a/virtual-banker/deployment/Dockerfile.widget b/virtual-banker/deployment/Dockerfile.widget deleted file mode 100644 index 5ad494f..0000000 --- a/virtual-banker/deployment/Dockerfile.widget +++ /dev/null @@ -1,28 +0,0 @@ -FROM node:20-alpine AS builder - -WORKDIR /app - -# Copy package files -COPY package*.json ./ -RUN npm ci - -# Copy source -COPY . . - -# Build -RUN npm run build - -# Production stage -FROM nginx:alpine - -# Copy built files -COPY --from=builder /app/dist /usr/share/nginx/html -COPY --from=builder /app/public/widget.js /usr/share/nginx/html/widget.js - -# Copy nginx config -COPY nginx.conf /etc/nginx/conf.d/default.conf - -EXPOSE 80 - -CMD ["nginx", "-g", "daemon off;"] - diff --git a/virtual-banker/deployment/docker-compose.yml b/virtual-banker/deployment/docker-compose.yml deleted file mode 100644 index 4b06745..0000000 --- a/virtual-banker/deployment/docker-compose.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: '3.8' - -services: - virtual-banker-api: - build: - context: ../backend - dockerfile: ../deployment/Dockerfile.backend - environment: - - DATABASE_URL=postgres://explorer:changeme@postgres:5432/explorer?sslmode=disable - - REDIS_URL=redis://redis:6379 - - PORT=8081 - ports: - - "8081:8081" - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8081/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - deploy: - resources: - limits: - cpus: '2' - memory: 2G - reservations: - cpus: '1' - memory: 1G - restart: unless-stopped - labels: - - "com.solacescanscout.name=virtual-banker-api" - - "com.solacescanscout.version=1.0.0" - - "com.solacescanscout.service=virtual-banker-api" - - virtual-banker-widget: - build: - context: ../widget - dockerfile: ../deployment/Dockerfile.widget - ports: - - "8082:80" - restart: unless-stopped - labels: - - "com.solacescanscout.name=virtual-banker-widget" - - "com.solacescanscout.version=1.0.0" - - "com.solacescanscout.service=virtual-banker-widget-cdn" - diff --git a/virtual-banker/deployment/nginx.conf b/virtual-banker/deployment/nginx.conf deleted file mode 100644 index aa2a50f..0000000 --- a/virtual-banker/deployment/nginx.conf +++ /dev/null @@ -1,44 +0,0 @@ -server { - listen 80; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # Enable gzip - gzip on; - gzip_vary on; - gzip_min_length 1024; - gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml+rss application/json application/javascript; - - # Security headers - add_header X-Frame-Options "SAMEORIGIN" always; - add_header X-Content-Type-Options "nosniff" always; - add_header X-XSS-Protection "1; mode=block" always; - - # CORS for widget embedding - add_header Access-Control-Allow-Origin "*" always; - add_header Access-Control-Allow-Methods "GET, POST, OPTIONS" always; - add_header Access-Control-Allow-Headers "Content-Type, Authorization" always; - - # Widget loader script - location /widget.js { - add_header Content-Type "application/javascript"; - expires 1h; - } - - # Static assets - location / { - try_files $uri $uri/ /index.html; - expires 1h; - add_header Cache-Control "public, immutable"; - } - - # Health check - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } -} - diff --git a/virtual-banker/docs/API.md b/virtual-banker/docs/API.md deleted file mode 100644 index 738edbe..0000000 --- a/virtual-banker/docs/API.md +++ /dev/null @@ -1,140 +0,0 @@ -# Virtual Banker API Reference - -## Base URL - -``` -http://localhost:8081 -``` - -## Authentication - -All requests (except health check) require authentication via JWT token in the `Authorization` header: - -``` -Authorization: Bearer -``` - -## Endpoints - -### Health Check - -``` -GET /health -``` - -**Response:** -```json -{ - "status": "healthy" -} -``` - -### Create Session - -``` -POST /v1/sessions -``` - -**Request Body:** -```json -{ - "tenant_id": "tenant-123", - "user_id": "user-456", - "auth_assertion": "jwt-token", - "portal_context": { - "route": "/account", - "account_id": "acc-789" - } -} -``` - -**Response:** -```json -{ - "session_id": "sess-abc123", - "ephemeral_token": "ephemeral-token-xyz", - "config": { - "theme": { - "primaryColor": "#0066cc" - }, - "avatar_enabled": true, - "greeting": "Hello! How can I help you today?", - "allowed_tools": ["get_account_status", "create_ticket"], - "policy": { - "max_session_duration_minutes": 30, - "rate_limit_per_minute": 10, - "require_consent": true - } - }, - "expires_at": "2024-01-20T15:30:00Z" -} -``` - -### Refresh Token - -``` -POST /v1/sessions/{session_id}/refresh-token -``` - -**Response:** -```json -{ - "ephemeral_token": "new-ephemeral-token", - "expires_at": "2024-01-20T15:35:00Z" -} -``` - -### End Session - -``` -POST /v1/sessions/{session_id}/end -``` - -**Response:** -```json -{ - "status": "ended" -} -``` - -## Error Responses - -All errors follow this format: - -```json -{ - "error": "Error message", - "message": "Detailed error description" -} -``` - -### Status Codes - -- `200 OK` - Success -- `201 Created` - Resource created -- `400 Bad Request` - Invalid request -- `401 Unauthorized` - Authentication required -- `404 Not Found` - Resource not found -- `500 Internal Server Error` - Server error - -## WebRTC Signaling - -WebRTC signaling is handled via WebSocket (to be implemented in Phase 1): - -``` -WS /v1/realtime/{session_id} -``` - -## Rate Limiting - -Rate limits are enforced per tenant and user: -- Default: 10 requests per minute per user -- Configurable per tenant - -Rate limit headers: -``` -X-RateLimit-Limit: 10 -X-RateLimit-Remaining: 9 -X-RateLimit-Reset: 1642680000 -``` - diff --git a/virtual-banker/docs/ARCHITECTURE.md b/virtual-banker/docs/ARCHITECTURE.md deleted file mode 100644 index 8daf065..0000000 --- a/virtual-banker/docs/ARCHITECTURE.md +++ /dev/null @@ -1,166 +0,0 @@ -# Virtual Banker Architecture - -## Overview - -The Virtual Banker is a multi-layered system that provides a digital human banking experience with full video realism, real-time voice interaction, and embeddable widget capabilities. - -## System Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Client Layer │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ Embeddable Widget (React/TypeScript) │ │ -│ │ - Chat UI │ │ -│ │ - Voice Controls │ │ -│ │ - Avatar View │ │ -│ │ - WebRTC Client │ │ -│ └──────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Edge Layer │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ CDN │ │ API Gateway │ │ WebRTC │ │ -│ │ (Widget) │ │ (Auth/Rate) │ │ Gateway │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Core Services │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Session │ │ Orchestrator │ │ LLM Gateway │ │ -│ │ Service │ │ │ │ │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ RAG Service │ │ Tool/Action │ │ Safety/ │ │ -│ │ │ │ Service │ │ Compliance │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Media Services │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ ASR Service │ │ TTS Service │ │ Avatar │ │ -│ │ (Streaming) │ │ (Streaming) │ │ Renderer │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Data Layer │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ PostgreSQL │ │ Redis │ │ Vector DB │ │ -│ │ (State) │ │ (Cache) │ │ (pgvector) │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -## Data Flow - -### Voice Turn Flow - -1. **User speaks** → Widget captures audio via microphone -2. **Audio stream** → WebRTC gateway → ASR service -3. **ASR** → Transcribes to text (partial + final) -4. **Orchestrator** → Sends transcript to LLM with context -5. **LLM** → Generates response + tool calls + emotion tags -6. **TTS** → Converts text to audio stream -7. **Avatar** → Generates visemes, expressions, gestures -8. **Widget** → Plays audio, displays captions, animates avatar - -### Text Turn Flow - -1. **User types** → Widget sends text message -2. **Orchestrator** → Processes message (same as step 4+ above) - -## Components - -### Backend Services - -#### Session Service -- Creates and manages sessions -- Issues ephemeral tokens -- Loads tenant configurations -- Tracks session state - -#### Conversation Orchestrator -- Maintains conversation state machine -- Routes messages to appropriate services -- Handles barge-in (interruptions) -- Synchronizes audio/video - -#### LLM Gateway -- Multi-tenant prompt templates -- Function/tool calling -- Output schema enforcement -- Model routing - -#### RAG Service -- Document ingestion and embedding -- Vector similarity search -- Reranking -- Citation formatting - -#### Tool/Action Service -- Tool registry and execution -- Banking service integrations -- Human-in-the-loop confirmations -- Audit logging - -### Frontend Widget - -#### Components -- **ChatPanel**: Main chat interface -- **VoiceControls**: Push-to-talk, hands-free, volume -- **AvatarView**: Video stream display -- **Captions**: Real-time captions overlay -- **Settings**: User preferences - -#### Hooks -- **useSession**: Session management -- **useConversation**: Message handling -- **useWebRTC**: WebRTC connection - -### Avatar System - -#### Unreal Engine -- Digital human character -- Blendshapes for visemes/expressions -- Animation blueprints -- PixelStreaming for video output - -#### Render Service -- Controls Unreal instances -- Manages GPU resources -- Streams video via WebRTC - -## Security - -- JWT/SSO authentication -- Ephemeral session tokens -- PII redaction -- Content filtering -- Rate limiting -- Audit trails - -## Accessibility - -- WCAG 2.1 AA compliance -- Keyboard navigation -- Screen reader support -- Captions (always available) -- Reduced motion support -- ARIA labels - -## Scalability - -- Stateless services (behind load balancer) -- Redis for session caching -- PostgreSQL for persistent state -- GPU cluster for avatar rendering -- CDN for widget assets - diff --git a/virtual-banker/docs/DEPLOYMENT.md b/virtual-banker/docs/DEPLOYMENT.md deleted file mode 100644 index c90812e..0000000 --- a/virtual-banker/docs/DEPLOYMENT.md +++ /dev/null @@ -1,213 +0,0 @@ -# Deployment Guide - -## Prerequisites - -- Docker and Docker Compose -- PostgreSQL 16+ with pgvector extension -- Redis 7+ -- (Optional) Kubernetes cluster for production - -## Development Deployment - -### 1. Database Setup - -Run migrations: - -```bash -cd virtual-banker/database -psql -U explorer -d explorer -f migrations/001_sessions.up.sql -psql -U explorer -d explorer -f migrations/002_conversations.up.sql -psql -U explorer -d explorer -f migrations/003_tenants.up.sql -psql -U explorer -d explorer -f migrations/004_vector_extension.up.sql -``` - -### 2. Start Services - -Using the main docker-compose.yml: - -```bash -cd deployment -docker-compose up -d virtual-banker-api virtual-banker-widget -``` - -Or using the virtual-banker specific compose file: - -```bash -cd virtual-banker/deployment -docker-compose up -d -``` - -### 3. Verify - -Check health: - -```bash -curl http://localhost:8081/health -``` - -Access widget: - -``` -http://localhost:8082 -``` - -## Production Deployment - -### Environment Variables - -**Backend API:** -```bash -DATABASE_URL=postgres://user:pass@host:5432/db -REDIS_URL=redis://host:6379 -PORT=8081 -``` - -**Widget CDN:** -- Deploy to CDN (Cloudflare, AWS CloudFront, etc.) -- Configure CORS headers -- Enable gzip compression - -### Docker Compose Production - -```yaml -services: - virtual-banker-api: - image: your-registry/virtual-banker-api:latest - environment: - - DATABASE_URL=${DATABASE_URL} - - REDIS_URL=${REDIS_URL} - deploy: - replicas: 3 - resources: - limits: - cpus: '2' - memory: 2G -``` - -### Kubernetes Deployment - -Example deployment: - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: virtual-banker-api -spec: - replicas: 3 - selector: - matchLabels: - app: virtual-banker-api - template: - metadata: - labels: - app: virtual-banker-api - spec: - containers: - - name: api - image: your-registry/virtual-banker-api:latest - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: virtual-banker-secrets - key: database-url - ports: - - containerPort: 8081 - resources: - requests: - memory: "1Gi" - cpu: "1" - limits: - memory: "2Gi" - cpu: "2" -``` - -## Scaling - -### Horizontal Scaling - -- **API Service**: Stateless, scale horizontally -- **Widget CDN**: Use CDN for global distribution -- **Avatar Renderer**: GPU-bound, scale based on concurrent sessions - -### Vertical Scaling - -- **Database**: Increase connection pool, add read replicas -- **Redis**: Use Redis Cluster for high availability -- **Avatar Renderer**: Allocate more GPU resources - -## Monitoring - -### Health Checks - -- API: `GET /health` -- Widget: `GET /health` (nginx) - -### Metrics - -- Session creation rate -- Active sessions -- API latency -- Error rates -- Avatar render queue - -### Logging - -- Structured logging (JSON) -- Log aggregation (ELK, Loki, etc.) -- Audit logs for compliance - -## Security - -### Network - -- Use internal networks for service communication -- Expose only necessary ports -- Use TLS for external communication - -### Secrets - -- Store secrets in secret management (Vault, AWS Secrets Manager) -- Rotate tokens regularly -- Use ephemeral tokens for sessions - -### Compliance - -- Enable audit logging -- Implement data retention policies -- PII redaction in logs -- Encryption at rest and in transit - -## Backup & Recovery - -### Database - -- Regular PostgreSQL backups -- Point-in-time recovery -- Test restore procedures - -### Redis - -- Enable persistence (AOF/RDB) -- Regular snapshots - -## Troubleshooting - -### Common Issues - -**Session creation fails:** -- Check database connection -- Verify tenant exists -- Check JWT validation - -**Widget not loading:** -- Check CORS configuration -- Verify CDN is accessible -- Check browser console for errors - -**Avatar not displaying:** -- Verify WebRTC connection -- Check avatar renderer service -- Verify GPU resources available - diff --git a/virtual-banker/docs/WIDGET_INTEGRATION.md b/virtual-banker/docs/WIDGET_INTEGRATION.md deleted file mode 100644 index 59181e7..0000000 --- a/virtual-banker/docs/WIDGET_INTEGRATION.md +++ /dev/null @@ -1,158 +0,0 @@ -# Widget Integration Guide - -## Quick Start - -### 1. Include the Widget Script - -Add the widget loader script to your HTML page: - -```html - -
-``` - -### 2. Configuration Options - -| Attribute | Required | Description | -|-----------|----------|-------------| -| `data-tenant-id` | Yes | Tenant identifier | -| `data-user-id` | No | User identifier (for authenticated sessions) | -| `data-auth-token` | No | JWT token for authentication | -| `data-api-url` | No | API base URL (default: http://localhost:8081) | -| `data-avatar-enabled` | No | Enable/disable avatar (default: true) | - -## Programmatic API - -### Methods - -```javascript -// Open widget -window.VirtualBankerWidgetAPI.open(); - -// Close widget -window.VirtualBankerWidgetAPI.close(); - -// Minimize widget -window.VirtualBankerWidgetAPI.minimize(); - -// Set context (page/route information) -window.VirtualBankerWidgetAPI.setContext({ - route: '/account', - accountId: 'acc-123', - productId: 'prod-456' -}); - -// Update authentication token -window.VirtualBankerWidgetAPI.setAuthToken('new-jwt-token'); -``` - -## PostMessage Events - -Listen for widget events from the parent window: - -```javascript -window.addEventListener('message', (event) => { - if (event.data.source === 'virtual-banker-widget') { - switch (event.data.type) { - case 'ready': - console.log('Widget is ready'); - break; - - case 'session_started': - console.log('Session ID:', event.data.payload.sessionId); - break; - - case 'action_requested': - console.log('Action:', event.data.payload.action); - console.log('Params:', event.data.payload.params); - // Handle action (e.g., open ticket, schedule appointment) - break; - - case 'action_completed': - console.log('Action completed:', event.data.payload); - break; - - case 'handoff_to_human': - console.log('Handoff reason:', event.data.payload.reason); - // Show human agent interface - break; - } - } -}); -``` - -## Sending Messages to Widget - -Send commands to the widget from the parent window: - -```javascript -const widget = document.getElementById('virtual-banker-widget'); -if (widget && widget.contentWindow) { - widget.contentWindow.postMessage({ - type: 'open', - source: 'virtual-banker-host' - }, '*'); -} -``` - -Available commands: -- `open` - Open the widget -- `close` - Close the widget -- `minimize` - Minimize the widget -- `setContext` - Update context -- `setAuthToken` - Update auth token - -## Styling - -The widget can be styled via CSS: - -```css -#virtual-banker-widget { - position: fixed; - bottom: 20px; - right: 20px; - width: 400px; - height: 600px; - z-index: 9999; - border-radius: 8px; - box-shadow: 0 4px 16px rgba(0, 0, 0, 0.2); -} - -#virtual-banker-widget.minimized { - height: 60px; - width: 200px; -} -``` - -## Theming - -Theme can be configured per tenant via the backend API. The widget will automatically apply the theme from the session configuration. - -## Accessibility - -The widget is built with accessibility in mind: -- Full keyboard navigation -- Screen reader support -- ARIA labels -- Captions always available -- Reduced motion support (respects OS preference) - -## Browser Support - -- Chrome/Edge 90+ -- Firefox 88+ -- Safari 14+ -- Mobile browsers (iOS Safari, Chrome Mobile) - -## Security - -- Content Security Policy (CSP) compatible -- No direct secrets in browser -- Ephemeral session tokens only -- CORS configured for embedding - diff --git a/virtual-banker/docs/openapi.yaml b/virtual-banker/docs/openapi.yaml deleted file mode 100644 index 76a30b5..0000000 --- a/virtual-banker/docs/openapi.yaml +++ /dev/null @@ -1,1137 +0,0 @@ -openapi: 3.0.3 -info: - title: Virtual Banker API - version: 1.0.0 - description: | - REST API for the Virtual Banker platform, including voice operations (TTS/ASR), - session management, conversation orchestration, and avatar control. - - Features: - - Voice session management - - Text-to-speech (TTS) synthesis with viseme support - - Speech-to-text (ASR) transcription - - Conversation orchestration - - WebRTC signaling for real-time communication - - Avatar animation and lip sync - - contact: - name: Virtual Banker Support - email: support@d-bis.org - license: - name: MIT - url: https://opensource.org/licenses/MIT - -servers: - - url: https://virtual-banker.d-bis.org/v1 - description: Production server - - url: https://sandbox-virtual-banker.d-bis.org/v1 - description: Sandbox server - - url: http://localhost:8081/v1 - description: Development server - -security: - - BearerAuth: [] - -tags: - - name: Sessions - description: Session management operations - - name: Voice - description: Voice operations (TTS/ASR) - - name: Conversation - description: Conversation orchestration - - name: Avatar - description: Avatar control and animation - - name: Providers - description: Voice provider management - - name: Health - description: Health check endpoints - -paths: - /health: - get: - tags: [Health] - summary: Health check - description: Returns the health status of the Virtual Banker API and service connections - operationId: getHealth - security: [] - responses: - '200': - description: Service is healthy - content: - application/json: - schema: - type: object - properties: - status: - type: string - example: "healthy" - services: - type: object - properties: - asr: - type: object - properties: - provider: - type: string - example: "mock" - status: - type: string - example: "available" - tts: - type: object - properties: - provider: - type: string - example: "mock" - status: - type: string - example: "available" - llm: - type: object - properties: - provider: - type: string - example: "mock" - status: - type: string - example: "available" - database: - type: string - example: "connected" - redis: - type: string - example: "connected" - timestamp: - type: string - format: date-time - - /sessions: - post: - tags: [Sessions] - summary: Create session - description: Creates a new virtual banker session - operationId: createSession - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateSessionRequest' - example: - tenant_id: "tenant-123" - user_id: "user-456" - auth_assertion: "jwt-token" - portal_context: - route: "/account" - account_id: "acc-789" - responses: - '201': - description: Session created successfully - content: - application/json: - schema: - $ref: '#/components/schemas/SessionResponse' - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '500': - $ref: '#/components/responses/InternalServerError' - - get: - tags: [Sessions] - summary: List sessions - description: Returns a list of active sessions for the authenticated user - operationId: listSessions - responses: - '200': - description: List of sessions - content: - application/json: - schema: - $ref: '#/components/schemas/SessionListResponse' - '401': - $ref: '#/components/responses/Unauthorized' - '500': - $ref: '#/components/responses/InternalServerError' - - /sessions/{sessionId}: - get: - tags: [Sessions] - summary: Get session - description: Returns session details - operationId: getSession - parameters: - - $ref: '#/components/parameters/SessionId' - responses: - '200': - description: Session details - content: - application/json: - schema: - $ref: '#/components/schemas/SessionResponse' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - delete: - tags: [Sessions] - summary: End session - description: Ends a virtual banker session - operationId: endSession - parameters: - - $ref: '#/components/parameters/SessionId' - responses: - '200': - description: Session ended - content: - application/json: - schema: - $ref: '#/components/schemas/BaseResponse' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /sessions/{sessionId}/refresh-token: - post: - tags: [Sessions] - summary: Refresh ephemeral token - description: Refreshes the ephemeral token for a session - operationId: refreshToken - parameters: - - $ref: '#/components/parameters/SessionId' - responses: - '200': - description: Token refreshed - content: - application/json: - schema: - type: object - properties: - success: - type: boolean - data: - type: object - properties: - ephemeral_token: - type: string - expires_at: - type: string - format: date-time - timestamp: - type: string - format: date-time - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/sessions: - post: - tags: [Voice] - summary: Create voice session - description: Creates a voice session for real-time TTS/ASR operations - operationId: createVoiceSession - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateVoiceSessionRequest' - example: - session_id: "sess-abc123" - provider: - asr: "deepgram" - tts: "elevenlabs" - voice_config: - voice_id: "21m00Tcm4TlvDq8ikWAM" - model_id: "eleven_multilingual_v2" - stability: 0.5 - similarity_boost: 0.75 - responses: - '201': - description: Voice session created - content: - application/json: - schema: - $ref: '#/components/schemas/VoiceSessionResponse' - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/sessions/{sessionId}/transcribe: - post: - tags: [Voice] - summary: Transcribe audio (ASR) - description: Transcribes audio to text using speech-to-text service - operationId: transcribeAudio - parameters: - - $ref: '#/components/parameters/SessionId' - requestBody: - required: true - content: - multipart/form-data: - schema: - type: object - required: - - audio - properties: - audio: - type: string - format: binary - description: Audio file (WAV, MP3, OGG, etc.) - language: - type: string - description: Language code (optional, auto-detect if not provided) - example: "en-US" - format: - type: string - enum: [wav, mp3, ogg, webm] - default: "wav" - include_words: - type: boolean - description: Include word-level timestamps - default: false - content: - application/json: - schema: - type: object - required: - - audio_data - properties: - audio_data: - type: string - format: byte - description: Base64-encoded audio data - language: - type: string - example: "en-US" - include_words: - type: boolean - default: false - responses: - '200': - description: Transcription result - content: - application/json: - schema: - $ref: '#/components/schemas/TranscriptionResponse' - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/sessions/{sessionId}/transcribe-stream: - post: - tags: [Voice] - summary: Stream transcription (ASR) - description: Streams audio for real-time transcription (returns SSE stream) - operationId: transcribeAudioStream - parameters: - - $ref: '#/components/parameters/SessionId' - requestBody: - required: true - content: - multipart/form-data: - schema: - type: object - required: - - audio_stream - properties: - audio_stream: - type: string - format: binary - description: Audio stream - language: - type: string - example: "en-US" - responses: - '200': - description: Transcription event stream (SSE) - content: - text/event-stream: - schema: - type: string - example: | - event: partial - data: {"type":"partial","text":"Hello, how can I","confidence":0.95,"timestamp":1704067200} - - event: final - data: {"type":"final","text":"Hello, how can I help you today?","confidence":0.98,"timestamp":1704067210,"words":[{"word":"Hello","start_time":0.0,"end_time":0.5},{"word":"how","start_time":0.6,"end_time":0.8}]} - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/sessions/{sessionId}/synthesize: - post: - tags: [Voice] - summary: Synthesize speech (TTS) - description: Converts text to speech audio - operationId: synthesizeSpeech - parameters: - - $ref: '#/components/parameters/SessionId' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/SynthesizeRequest' - example: - text: "Hello, how can I help you today?" - voice_config: - voice_id: "21m00Tcm4TlvDq8ikWAM" - model_id: "eleven_multilingual_v2" - stability: 0.5 - similarity_boost: 0.75 - format: "mp3" - sample_rate: 44100 - responses: - '200': - description: Audio synthesis result - content: - application/json: - schema: - type: object - properties: - success: - type: boolean - data: - type: object - properties: - audio_data: - type: string - format: byte - description: Base64-encoded audio data - format: - type: string - example: "mp3" - sample_rate: - type: integer - example: 44100 - duration: - type: number - format: float - example: 2.5 - visemes: - type: array - items: - $ref: '#/components/schemas/VisemeEvent' - timestamp: - type: string - format: date-time - content: - audio/mpeg: - schema: - type: string - format: binary - audio/wav: - schema: - type: string - format: binary - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/sessions/{sessionId}/synthesize-stream: - post: - tags: [Voice] - summary: Stream speech synthesis (TTS) - description: Streams text for real-time speech synthesis (returns audio stream) - operationId: synthesizeSpeechStream - parameters: - - $ref: '#/components/parameters/SessionId' - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - text - properties: - text: - type: string - example: "Hello, how can I help you today?" - voice_config: - $ref: '#/components/schemas/VoiceConfig' - format: - type: string - enum: [mp3, wav, pcm] - default: "mp3" - sample_rate: - type: integer - enum: [16000, 22050, 44100] - default: 44100 - responses: - '200': - description: Audio stream - content: - audio/mpeg: - schema: - type: string - format: binary - audio/wav: - schema: - type: string - format: binary - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/sessions/{sessionId}/visemes: - post: - tags: [Voice] - summary: Get viseme events - description: Returns viseme (lip shape) events for text, used for avatar lip sync - operationId: getVisemes - parameters: - - $ref: '#/components/parameters/SessionId' - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - text - properties: - text: - type: string - example: "Hello, how can I help you today?" - responses: - '200': - description: Viseme events - content: - application/json: - schema: - $ref: '#/components/schemas/VisemeResponse' - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/sessions/{sessionId}/status: - get: - tags: [Voice] - summary: Get voice session status - description: Returns the status of a voice session - operationId: getVoiceSessionStatus - parameters: - - $ref: '#/components/parameters/SessionId' - responses: - '200': - description: Voice session status - content: - application/json: - schema: - type: object - properties: - success: - type: boolean - data: - type: object - properties: - session_id: - type: string - state: - type: string - enum: [idle, listening, thinking, speaking] - asr_status: - type: string - enum: [active, inactive, error] - tts_status: - type: string - enum: [active, inactive, error] - provider: - type: object - properties: - asr: - type: string - tts: - type: string - timestamp: - type: string - format: date-time - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/providers: - get: - tags: [Providers] - summary: List voice providers - description: Returns a list of available TTS and ASR providers - operationId: listVoiceProviders - responses: - '200': - description: List of providers - content: - application/json: - schema: - $ref: '#/components/schemas/ProviderListResponse' - '401': - $ref: '#/components/responses/Unauthorized' - '500': - $ref: '#/components/responses/InternalServerError' - - /voice/webhooks: - post: - tags: [Voice] - summary: Register webhook - description: Registers a webhook URL for voice session events - operationId: registerVoiceWebhook - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterWebhookRequest' - example: - url: "https://api.example.com/webhooks/voice" - events: - - "transcription.complete" - - "synthesis.complete" - - "session.state_changed" - secret: "webhook_secret_token" - responses: - '201': - description: Webhook registered - content: - application/json: - schema: - $ref: '#/components/schemas/WebhookResponse' - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - '500': - $ref: '#/components/responses/InternalServerError' - - /realtime/{sessionId}: - get: - tags: [Conversation] - summary: WebRTC signaling endpoint - description: WebSocket endpoint for WebRTC signaling (SDP exchange, ICE candidates) - operationId: webrtcSignaling - parameters: - - $ref: '#/components/parameters/SessionId' - responses: - '101': - description: Switching protocols to WebSocket - headers: - Upgrade: - schema: - type: string - example: "websocket" - Connection: - schema: - type: string - example: "Upgrade" - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '404': - $ref: '#/components/responses/NotFound' - -components: - securitySchemes: - BearerAuth: - type: http - scheme: bearer - bearerFormat: JWT - description: JWT token or ephemeral session token - - parameters: - SessionId: - name: sessionId - in: path - required: true - description: Session ID - schema: - type: string - example: "sess-abc123" - - schemas: - CreateSessionRequest: - type: object - required: - - tenant_id - - user_id - - auth_assertion - properties: - tenant_id: - type: string - description: Tenant identifier - example: "tenant-123" - user_id: - type: string - description: User identifier - example: "user-456" - auth_assertion: - type: string - description: JWT authentication token - example: "jwt-token" - portal_context: - type: object - description: Portal context for session - properties: - route: - type: string - example: "/account" - account_id: - type: string - example: "acc-789" - - CreateVoiceSessionRequest: - type: object - required: - - session_id - properties: - session_id: - type: string - description: Parent session ID - example: "sess-abc123" - provider: - type: object - properties: - asr: - type: string - enum: [mock, deepgram, google] - default: "mock" - example: "deepgram" - tts: - type: string - enum: [mock, elevenlabs, azure] - default: "mock" - example: "elevenlabs" - voice_config: - $ref: '#/components/schemas/VoiceConfig' - - VoiceConfig: - type: object - properties: - voice_id: - type: string - description: Voice ID (provider-specific) - example: "21m00Tcm4TlvDq8ikWAM" - model_id: - type: string - description: Model ID (provider-specific) - example: "eleven_multilingual_v2" - stability: - type: number - format: float - description: Stability parameter (0.0-1.0) - minimum: 0.0 - maximum: 1.0 - example: 0.5 - similarity_boost: - type: number - format: float - description: Similarity boost parameter (0.0-1.0) - minimum: 0.0 - maximum: 1.0 - example: 0.75 - style: - type: number - format: float - description: Style parameter (0.0-1.0, ElevenLabs) - minimum: 0.0 - maximum: 1.0 - use_speaker_boost: - type: boolean - description: Enable speaker boost (ElevenLabs) - - SynthesizeRequest: - type: object - required: - - text - properties: - text: - type: string - description: Text to synthesize - example: "Hello, how can I help you today?" - voice_config: - $ref: '#/components/schemas/VoiceConfig' - format: - type: string - enum: [mp3, wav, pcm] - default: "mp3" - example: "mp3" - sample_rate: - type: integer - enum: [16000, 22050, 44100] - default: 44100 - example: 44100 - - RegisterWebhookRequest: - type: object - required: - - url - - events - properties: - url: - type: string - format: uri - description: Webhook URL - example: "https://api.example.com/webhooks/voice" - events: - type: array - description: Events to subscribe to - items: - type: string - enum: [transcription.complete, transcription.partial, synthesis.complete, session.state_changed, session.created, session.ended] - example: ["transcription.complete", "synthesis.complete"] - secret: - type: string - description: Webhook secret for signature verification - example: "webhook_secret_token" - active: - type: boolean - default: true - - Session: - type: object - properties: - session_id: - type: string - tenant_id: - type: string - user_id: - type: string - ephemeral_token: - type: string - config: - type: object - properties: - theme: - type: object - properties: - primaryColor: - type: string - avatar_enabled: - type: boolean - greeting: - type: string - allowed_tools: - type: array - items: - type: string - policy: - type: object - properties: - max_session_duration_minutes: - type: integer - rate_limit_per_minute: - type: integer - require_consent: - type: boolean - expires_at: - type: string - format: date-time - created_at: - type: string - format: date-time - - SessionResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - data: - $ref: '#/components/schemas/Session' - - SessionListResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - data: - type: object - properties: - sessions: - type: array - items: - $ref: '#/components/schemas/Session' - - VoiceSession: - type: object - properties: - session_id: - type: string - state: - type: string - enum: [idle, listening, thinking, speaking] - provider: - type: object - properties: - asr: - type: string - tts: - type: string - voice_config: - $ref: '#/components/schemas/VoiceConfig' - created_at: - type: string - format: date-time - - VoiceSessionResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - data: - $ref: '#/components/schemas/VoiceSession' - - TranscriptionEvent: - type: object - properties: - type: - type: string - enum: [partial, final] - text: - type: string - confidence: - type: number - format: float - minimum: 0.0 - maximum: 1.0 - timestamp: - type: integer - format: int64 - words: - type: array - items: - $ref: '#/components/schemas/Word' - - Word: - type: object - properties: - word: - type: string - start_time: - type: number - format: float - end_time: - type: number - format: float - confidence: - type: number - format: float - minimum: 0.0 - maximum: 1.0 - - TranscriptionResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - data: - $ref: '#/components/schemas/TranscriptionEvent' - - VisemeEvent: - type: object - properties: - viseme: - type: string - description: Viseme identifier (e.g., "sil", "aa", "ee", "oh", "ou") - example: "aa" - start_time: - type: number - format: float - description: Start time in seconds - example: 0.1 - end_time: - type: number - format: float - description: End time in seconds - example: 0.3 - phoneme: - type: string - description: Phoneme identifier (optional) - example: "/a/" - - VisemeResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - data: - type: object - properties: - text: - type: string - visemes: - type: array - items: - $ref: '#/components/schemas/VisemeEvent' - duration: - type: number - format: float - - Provider: - type: object - properties: - id: - type: string - name: - type: string - type: - type: string - enum: [asr, tts] - available: - type: boolean - configured: - type: boolean - features: - type: array - items: - type: string - example: ["streaming", "word_timestamps", "custom_voice"] - - ProviderListResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - data: - type: object - properties: - providers: - type: array - items: - $ref: '#/components/schemas/Provider' - - Webhook: - type: object - properties: - webhook_id: - type: string - url: - type: string - events: - type: array - items: - type: string - active: - type: boolean - created_at: - type: string - format: date-time - - WebhookResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - data: - $ref: '#/components/schemas/Webhook' - - BaseResponse: - type: object - properties: - success: - type: boolean - example: true - timestamp: - type: string - format: date-time - - ErrorResponse: - type: object - properties: - success: - type: boolean - example: false - error: - type: object - properties: - code: - type: string - example: "VALIDATION_ERROR" - message: - type: string - example: "Invalid request parameters" - details: - type: object - timestamp: - type: string - format: date-time - - responses: - BadRequest: - description: Bad request - validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - - Unauthorized: - description: Unauthorized - missing or invalid authentication - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - - Forbidden: - description: Forbidden - insufficient permissions - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - - NotFound: - description: Resource not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - - InternalServerError: - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' diff --git a/virtual-banker/scripts/setup-database.sh b/virtual-banker/scripts/setup-database.sh deleted file mode 100755 index 883bb0f..0000000 --- a/virtual-banker/scripts/setup-database.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Setup Virtual Banker Database -# This script runs all database migrations - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -DB_DIR="$PROJECT_ROOT/database/migrations" - -# Load environment variables -if [ -f "$PROJECT_ROOT/.env" ]; then - export $(cat "$PROJECT_ROOT/.env" | grep -v '^#' | xargs) -fi - -# Set defaults -export PGHOST="${PGHOST:-localhost}" -export PGPORT="${PGPORT:-5432}" -export PGUSER="${PGUSER:-explorer}" -export PGPASSWORD="${PGPASSWORD:-changeme}" -export PGDATABASE="${PGDATABASE:-explorer}" - -echo "Running database migrations..." - -# Run migrations in order -for migration in "$DB_DIR"/*.up.sql; do - if [ -f "$migration" ]; then - echo "Running $(basename $migration)..." - PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$migration" - fi -done - -echo "Database setup complete!" - diff --git a/virtual-banker/scripts/start-backend.sh b/virtual-banker/scripts/start-backend.sh deleted file mode 100755 index dc952d8..0000000 --- a/virtual-banker/scripts/start-backend.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Start Virtual Banker Backend Service -# This script starts the backend API server with proper environment configuration - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -BACKEND_DIR="$PROJECT_ROOT/backend" - -cd "$BACKEND_DIR" - -# Load environment variables -if [ -f "$PROJECT_ROOT/.env" ]; then - export $(cat "$PROJECT_ROOT/.env" | grep -v '^#' | xargs) -fi - -# Set defaults -export DATABASE_URL="${DATABASE_URL:-postgres://explorer:changeme@localhost:5432/explorer?sslmode=disable}" -export REDIS_URL="${REDIS_URL:-redis://localhost:6379}" -export PORT="${PORT:-8081}" - -echo "Starting Virtual Banker Backend..." -echo "Database: $DATABASE_URL" -echo "Redis: $REDIS_URL" -echo "Port: $PORT" - -# Run the service -go run main.go - diff --git a/virtual-banker/widget/package.json b/virtual-banker/widget/package.json deleted file mode 100644 index 902bf2e..0000000 --- a/virtual-banker/widget/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "@explorer/virtual-banker-widget", - "version": "1.0.0", - "description": "Embeddable Virtual Banker widget", - "main": "dist/widget.js", - "scripts": { - "build": "webpack --mode production", - "dev": "webpack --mode development --watch", - "type-check": "tsc --noEmit" - }, - "dependencies": { - "react": "^18.2.0", - "react-dom": "^18.2.0" - }, - "devDependencies": { - "@types/react": "^18.2.45", - "@types/react-dom": "^18.2.18", - "typescript": "^5.3.3", - "webpack": "^5.89.0", - "webpack-cli": "^5.1.4", - "ts-loader": "^9.5.1", - "css-loader": "^6.8.1", - "style-loader": "^3.3.3", - "html-webpack-plugin": "^5.5.3" - } -} - diff --git a/virtual-banker/widget/public/index.html b/virtual-banker/widget/public/index.html deleted file mode 100644 index 6e23e0f..0000000 --- a/virtual-banker/widget/public/index.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - Virtual Banker Widget - - -
- - - diff --git a/virtual-banker/widget/public/widget.js b/virtual-banker/widget/public/widget.js deleted file mode 100644 index bfdfd31..0000000 --- a/virtual-banker/widget/public/widget.js +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Virtual Banker Widget Loader - * - * Usage: - * - *
- */ - -(function() { - 'use strict'; - - // Get configuration from script tag - const script = document.currentScript; - const config = { - tenantId: script.getAttribute('data-tenant-id') || 'default', - userId: script.getAttribute('data-user-id') || undefined, - authToken: script.getAttribute('data-auth-token') || undefined, - apiUrl: script.getAttribute('data-api-url') || undefined, - avatarEnabled: script.getAttribute('data-avatar-enabled') !== 'false', - }; - - // Load React and ReactDOM (should be loaded separately or bundled) - // For now, this is a placeholder - the actual widget will be loaded via the built bundle - console.log('Virtual Banker Widget Loader initialized', config); - - // Create container if it doesn't exist - let container = document.getElementById('virtual-banker-widget'); - if (!container) { - container = document.createElement('div'); - container.id = 'virtual-banker-widget'; - document.body.appendChild(container); - } - - // Store config for widget initialization - container.dataset.tenantId = config.tenantId; - if (config.userId) container.dataset.userId = config.userId; - if (config.authToken) container.dataset.authToken = config.authToken; - if (config.apiUrl) container.dataset.apiUrl = config.apiUrl; - container.dataset.avatarEnabled = config.avatarEnabled.toString(); - - // Export API for programmatic control - window.VirtualBankerWidgetAPI = { - open: function() { - const widget = document.getElementById('virtual-banker-widget'); - if (widget) { - widget.style.display = 'block'; - } - }, - close: function() { - const widget = document.getElementById('virtual-banker-widget'); - if (widget) { - widget.style.display = 'none'; - } - }, - minimize: function() { - const widget = document.getElementById('virtual-banker-widget'); - if (widget) { - widget.classList.add('minimized'); - } - }, - setContext: function(context) { - const widget = document.getElementById('virtual-banker-widget'); - if (widget) { - widget.dataset.context = JSON.stringify(context); - } - }, - setAuthToken: function(token) { - const widget = document.getElementById('virtual-banker-widget'); - if (widget) { - widget.dataset.authToken = token; - } - }, - }; -})(); - diff --git a/virtual-banker/widget/src/App.css b/virtual-banker/widget/src/App.css deleted file mode 100644 index d5aae0c..0000000 --- a/virtual-banker/widget/src/App.css +++ /dev/null @@ -1,94 +0,0 @@ -.widget-container { - width: 400px; - height: 600px; - border: 1px solid #ddd; - border-radius: 8px; - display: flex; - flex-direction: column; - background: white; - box-shadow: 0 4px 16px rgba(0, 0, 0, 0.1); - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', - 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; -} - -.widget-container.loading, -.widget-container.error { - align-items: center; - justify-content: center; - padding: 24px; -} - -.loading-spinner { - font-size: 16px; - color: #666; -} - -.error-message { - color: #d32f2f; - margin-bottom: 16px; - text-align: center; -} - -.widget-header { - display: flex; - justify-content: space-between; - align-items: center; - padding: 16px; - border-bottom: 1px solid #e0e0e0; -} - -.widget-header h1 { - margin: 0; - font-size: 20px; - font-weight: 600; -} - -.settings-button { - background: none; - border: none; - font-size: 20px; - cursor: pointer; - padding: 4px 8px; - border-radius: 4px; - transition: background-color 0.2s; -} - -.settings-button:hover { - background-color: #f0f0f0; -} - -.widget-content { - flex: 1; - display: flex; - flex-direction: column; - overflow: hidden; -} - -.widget-avatar-section { - height: 200px; - padding: 8px; -} - -.widget-chat-section { - flex: 1; - min-height: 0; -} - -/* Responsive adjustments */ -@media (max-width: 480px) { - .widget-container { - width: 100%; - height: 100vh; - border-radius: 0; - } -} - -/* Reduced motion support */ -@media (prefers-reduced-motion: reduce) { - * { - animation-duration: 0.01ms !important; - animation-iteration-count: 1 !important; - transition-duration: 0.01ms !important; - } -} - diff --git a/virtual-banker/widget/src/App.tsx b/virtual-banker/widget/src/App.tsx deleted file mode 100644 index 25c6f74..0000000 --- a/virtual-banker/widget/src/App.tsx +++ /dev/null @@ -1,189 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { ChatPanel } from './components/ChatPanel'; -import { VoiceControls } from './components/VoiceControls'; -import { AvatarView } from './components/AvatarView'; -import { Captions } from './components/Captions'; -import { Settings } from './components/Settings'; -import { useSession } from './hooks/useSession'; -import { useConversation } from './hooks/useConversation'; -import { useWebRTC } from './hooks/useWebRTC'; -import { PostMessageAPI } from './services/postMessage'; -import { WidgetConfig } from './types'; -import './App.css'; - -// Default config - can be overridden via postMessage or data attributes -const getConfig = (): WidgetConfig => { - const script = document.querySelector('script[data-tenant-id]'); - if (script) { - return { - tenantId: script.getAttribute('data-tenant-id') || 'default', - userId: script.getAttribute('data-user-id') || undefined, - authToken: script.getAttribute('data-auth-token') || undefined, - apiUrl: script.getAttribute('data-api-url') || undefined, - avatarEnabled: script.getAttribute('data-avatar-enabled') !== 'false', - }; - } - - return { - tenantId: 'default', - avatarEnabled: true, - }; -}; - -export const App: React.FC = () => { - const [config] = useState(getConfig()); - const [showSettings, setShowSettings] = useState(false); - const [showCaptions, setShowCaptions] = useState(true); - const [avatarEnabled, setAvatarEnabled] = useState(config.avatarEnabled ?? true); - const [volume, setVolume] = useState(100); - const [isMuted, setIsMuted] = useState(false); - const [captionText, setCaptionText] = useState(''); - - const postMessage = new PostMessageAPI(); - const { session, loading, error, createSession, endSession } = useSession(config); - const { - messages, - isListening, - isSpeaking, - setIsListening, - setIsSpeaking, - sendMessage, - receiveMessage, - } = useConversation(); - const { isConnected, remoteStream, initializeWebRTC, closeWebRTC } = useWebRTC(); - - // Initialize session on mount - useEffect(() => { - createSession(); - }, []); - - // Initialize WebRTC when session is ready - useEffect(() => { - if (session && !isConnected) { - initializeWebRTC(); - } - }, [session, isConnected]); - - // Cleanup on unmount - useEffect(() => { - return () => { - endSession(); - closeWebRTC(); - }; - }, []); - - // Send ready event - useEffect(() => { - if (session) { - postMessage.ready(); - postMessage.sessionStarted(session.sessionId); - } - }, [session]); - - // Listen for messages from host - useEffect(() => { - const unsubscribe = postMessage.on('open', () => { - // Widget opened - }); - - return unsubscribe; - }, []); - - const handleSendMessage = (message: string) => { - sendMessage(message); - // TODO: Send to backend via WebRTC or WebSocket - }; - - const handlePushToTalk = () => { - setIsListening(true); - // TODO: Start audio capture - }; - - const handleHandsFree = () => { - setIsListening(true); - // TODO: Enable continuous listening - }; - - const handleToggleMute = () => { - setIsMuted(!isMuted); - // TODO: Mute/unmute audio - }; - - if (loading) { - return ( -
-
Loading...
-
- ); - } - - if (error) { - return ( -
-
Error: {error}
- -
- ); - } - - return ( -
-
-

Virtual Banker

- -
- -
- {avatarEnabled && ( -
- setAvatarEnabled(false)} - /> -
- )} - -
- setShowCaptions(!showCaptions)} - /> -
-
- - - - - - {showSettings && ( - setShowCaptions(!showCaptions)} - avatarEnabled={avatarEnabled} - onToggleAvatar={() => setAvatarEnabled(!avatarEnabled)} - onClose={() => setShowSettings(false)} - /> - )} -
- ); -}; - diff --git a/virtual-banker/widget/src/components/AvatarView.css b/virtual-banker/widget/src/components/AvatarView.css deleted file mode 100644 index 0242335..0000000 --- a/virtual-banker/widget/src/components/AvatarView.css +++ /dev/null @@ -1,60 +0,0 @@ -.avatar-view { - position: relative; - width: 100%; - height: 100%; - background-color: #000; - display: flex; - align-items: center; - justify-content: center; - border-radius: 8px; - overflow: hidden; -} - -.avatar-view.disabled { - background-color: #f5f5f5; -} - -.avatar-video { - width: 100%; - height: 100%; - object-fit: contain; -} - -.avatar-toggle { - position: absolute; - top: 8px; - right: 8px; - width: 32px; - height: 32px; - border-radius: 50%; - border: none; - background-color: rgba(0, 0, 0, 0.6); - color: white; - cursor: pointer; - font-size: 18px; - display: flex; - align-items: center; - justify-content: center; - transition: background-color 0.2s; -} - -.avatar-toggle:hover { - background-color: rgba(0, 0, 0, 0.8); -} - -.enable-avatar-button { - padding: 12px 24px; - background-color: #0066cc; - color: white; - border: none; - border-radius: 4px; - cursor: pointer; - font-size: 14px; - font-weight: 500; - transition: background-color 0.2s; -} - -.enable-avatar-button:hover { - background-color: #0052a3; -} - diff --git a/virtual-banker/widget/src/components/AvatarView.tsx b/virtual-banker/widget/src/components/AvatarView.tsx deleted file mode 100644 index afb652f..0000000 --- a/virtual-banker/widget/src/components/AvatarView.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import React, { useRef, useEffect } from 'react'; -import './AvatarView.css'; - -interface AvatarViewProps { - enabled: boolean; - videoStream?: MediaStream; - onToggle: () => void; -} - -export const AvatarView: React.FC = ({ - enabled, - videoStream, - onToggle, -}) => { - const videoRef = useRef(null); - - useEffect(() => { - if (videoRef.current && videoStream) { - videoRef.current.srcObject = videoStream; - } - }, [videoStream]); - - if (!enabled) { - return ( -
- -
- ); - } - - return ( -
-
- ); -}; - diff --git a/virtual-banker/widget/src/components/Captions.css b/virtual-banker/widget/src/components/Captions.css deleted file mode 100644 index a51fd9a..0000000 --- a/virtual-banker/widget/src/components/Captions.css +++ /dev/null @@ -1,17 +0,0 @@ -.captions { - position: fixed; - bottom: 80px; - left: 50%; - transform: translateX(-50%); - max-width: 80%; - padding: 12px 16px; - background-color: rgba(0, 0, 0, 0.8); - color: white; - border-radius: 4px; - font-size: 16px; - line-height: 1.5; - z-index: 1000; - text-align: center; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); -} - diff --git a/virtual-banker/widget/src/components/Captions.tsx b/virtual-banker/widget/src/components/Captions.tsx deleted file mode 100644 index c92e7bc..0000000 --- a/virtual-banker/widget/src/components/Captions.tsx +++ /dev/null @@ -1,20 +0,0 @@ -import React from 'react'; -import './Captions.css'; - -interface CaptionsProps { - text: string; - visible: boolean; -} - -export const Captions: React.FC = ({ text, visible }) => { - if (!visible || !text) { - return null; - } - - return ( -
- {text} -
- ); -}; - diff --git a/virtual-banker/widget/src/components/ChatPanel.css b/virtual-banker/widget/src/components/ChatPanel.css deleted file mode 100644 index b5d54ee..0000000 --- a/virtual-banker/widget/src/components/ChatPanel.css +++ /dev/null @@ -1,141 +0,0 @@ -.chat-panel { - display: flex; - flex-direction: column; - height: 100%; - background: #ffffff; - border-radius: 8px; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); -} - -.chat-header { - display: flex; - justify-content: space-between; - align-items: center; - padding: 16px; - border-bottom: 1px solid #e0e0e0; -} - -.chat-header h2 { - margin: 0; - font-size: 18px; - font-weight: 600; -} - -.caption-toggle { - background: none; - border: none; - font-size: 20px; - cursor: pointer; - padding: 4px 8px; - border-radius: 4px; - transition: background-color 0.2s; -} - -.caption-toggle:hover { - background-color: #f0f0f0; -} - -.chat-messages { - flex: 1; - overflow-y: auto; - padding: 16px; - display: flex; - flex-direction: column; - gap: 12px; -} - -.message { - display: flex; - flex-direction: column; - max-width: 80%; - padding: 12px; - border-radius: 8px; -} - -.message-user { - align-self: flex-end; - background-color: #0066cc; - color: white; -} - -.message-assistant { - align-self: flex-start; - background-color: #f0f0f0; - color: #333; -} - -.message-content { - word-wrap: break-word; - line-height: 1.5; -} - -.message-timestamp { - font-size: 11px; - opacity: 0.7; - margin-top: 4px; -} - -.chat-status { - padding: 8px 16px; - min-height: 32px; - display: flex; - align-items: center; -} - -.status-indicator { - font-size: 14px; - padding: 4px 8px; - border-radius: 4px; -} - -.status-indicator.listening { - background-color: #e3f2fd; - color: #1976d2; -} - -.status-indicator.speaking { - background-color: #fff3e0; - color: #f57c00; -} - -.chat-input-form { - display: flex; - padding: 16px; - border-top: 1px solid #e0e0e0; - gap: 8px; -} - -.chat-input { - flex: 1; - padding: 10px 12px; - border: 1px solid #ddd; - border-radius: 4px; - font-size: 14px; -} - -.chat-input:focus { - outline: none; - border-color: #0066cc; -} - -.send-button { - padding: 10px 20px; - background-color: #0066cc; - color: white; - border: none; - border-radius: 4px; - cursor: pointer; - font-size: 14px; - font-weight: 500; - transition: background-color 0.2s; -} - -.send-button:hover:not(:disabled) { - background-color: #0052a3; -} - -.send-button:disabled { - opacity: 0.5; - cursor: not-allowed; -} - diff --git a/virtual-banker/widget/src/components/ChatPanel.tsx b/virtual-banker/widget/src/components/ChatPanel.tsx deleted file mode 100644 index 5f5bf59..0000000 --- a/virtual-banker/widget/src/components/ChatPanel.tsx +++ /dev/null @@ -1,101 +0,0 @@ -import React, { useState, useRef, useEffect } from 'react'; -import { Message } from '../types'; -import './ChatPanel.css'; - -interface ChatPanelProps { - messages: Message[]; - onSendMessage: (message: string) => void; - isListening: boolean; - isSpeaking: boolean; - showCaptions: boolean; - onToggleCaptions: () => void; -} - -export const ChatPanel: React.FC = ({ - messages, - onSendMessage, - isListening, - isSpeaking, - showCaptions, - onToggleCaptions, -}) => { - const [input, setInput] = useState(''); - const messagesEndRef = useRef(null); - - useEffect(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); - }, [messages]); - - const handleSubmit = (e: React.FormEvent) => { - e.preventDefault(); - if (input.trim()) { - onSendMessage(input.trim()); - setInput(''); - } - }; - - return ( -
-
-

Virtual Banker

- -
- -
- {messages.map((message) => ( -
-
{message.content}
-
- {message.timestamp.toLocaleTimeString()} -
-
- ))} -
-
- -
- {isListening && ( - - 🎤 Listening... - - )} - {isSpeaking && ( - - 🔊 Speaking... - - )} -
- -
- setInput(e.target.value)} - placeholder="Type your message..." - className="chat-input" - aria-label="Message input" - disabled={isSpeaking} - /> - -
-
- ); -}; - diff --git a/virtual-banker/widget/src/components/Settings.css b/virtual-banker/widget/src/components/Settings.css deleted file mode 100644 index e41c880..0000000 --- a/virtual-banker/widget/src/components/Settings.css +++ /dev/null @@ -1,70 +0,0 @@ -.settings-overlay { - position: fixed; - top: 0; - left: 0; - right: 0; - bottom: 0; - background-color: rgba(0, 0, 0, 0.5); - display: flex; - align-items: center; - justify-content: center; - z-index: 2000; -} - -.settings-panel { - background-color: white; - border-radius: 8px; - padding: 24px; - max-width: 400px; - width: 90%; - box-shadow: 0 4px 16px rgba(0, 0, 0, 0.2); -} - -.settings-header { - display: flex; - justify-content: space-between; - align-items: center; - margin-bottom: 24px; -} - -.settings-header h2 { - margin: 0; - font-size: 20px; - font-weight: 600; -} - -.close-button { - background: none; - border: none; - font-size: 24px; - cursor: pointer; - padding: 4px; - line-height: 1; - color: #666; - transition: color 0.2s; -} - -.close-button:hover { - color: #000; -} - -.settings-content { - display: flex; - flex-direction: column; - gap: 16px; -} - -.setting-item label { - display: flex; - align-items: center; - gap: 12px; - cursor: pointer; - font-size: 16px; -} - -.setting-item input[type="checkbox"] { - width: 20px; - height: 20px; - cursor: pointer; -} - diff --git a/virtual-banker/widget/src/components/Settings.tsx b/virtual-banker/widget/src/components/Settings.tsx deleted file mode 100644 index 79d0519..0000000 --- a/virtual-banker/widget/src/components/Settings.tsx +++ /dev/null @@ -1,56 +0,0 @@ -import React, { useState } from 'react'; -import './Settings.css'; - -interface SettingsProps { - showCaptions: boolean; - onToggleCaptions: () => void; - avatarEnabled: boolean; - onToggleAvatar: () => void; - onClose: () => void; -} - -export const Settings: React.FC = ({ - showCaptions, - onToggleCaptions, - avatarEnabled, - onToggleAvatar, - onClose, -}) => { - return ( -
-
-
-

Settings

- -
- -
-
- -
- -
- -
-
-
-
- ); -}; - diff --git a/virtual-banker/widget/src/components/VoiceControls.css b/virtual-banker/widget/src/components/VoiceControls.css deleted file mode 100644 index b17140a..0000000 --- a/virtual-banker/widget/src/components/VoiceControls.css +++ /dev/null @@ -1,116 +0,0 @@ -.voice-controls { - padding: 16px; - border-top: 1px solid #e0e0e0; - background-color: #f9f9f9; -} - -.voice-mode-selector { - display: flex; - gap: 8px; - margin-bottom: 12px; -} - -.mode-button { - flex: 1; - padding: 8px 16px; - border: 1px solid #ddd; - background-color: white; - border-radius: 4px; - cursor: pointer; - font-size: 14px; - transition: all 0.2s; -} - -.mode-button:hover { - background-color: #f0f0f0; -} - -.mode-button.active { - background-color: #0066cc; - color: white; - border-color: #0066cc; -} - -.voice-controls-row { - display: flex; - align-items: center; - gap: 12px; -} - -.control-button { - padding: 8px 12px; - border: 1px solid #ddd; - background-color: white; - border-radius: 4px; - cursor: pointer; - font-size: 18px; - transition: background-color 0.2s; -} - -.control-button:hover { - background-color: #f0f0f0; -} - -.control-button.muted { - background-color: #ffebee; - border-color: #f44336; -} - -.volume-control { - display: flex; - align-items: center; - gap: 8px; - flex: 1; -} - -.volume-slider { - flex: 1; - height: 4px; - border-radius: 2px; - background: #ddd; - outline: none; -} - -.volume-slider::-webkit-slider-thumb { - appearance: none; - width: 16px; - height: 16px; - border-radius: 50%; - background: #0066cc; - cursor: pointer; -} - -.volume-slider::-moz-range-thumb { - width: 16px; - height: 16px; - border-radius: 50%; - background: #0066cc; - cursor: pointer; - border: none; -} - -.volume-value { - font-size: 12px; - color: #666; - min-width: 40px; - text-align: right; -} - -.listening-indicator { - font-size: 14px; - color: #1976d2; - font-weight: 500; -} - -.sr-only { - position: absolute; - width: 1px; - height: 1px; - padding: 0; - margin: -1px; - overflow: hidden; - clip: rect(0, 0, 0, 0); - white-space: nowrap; - border-width: 0; -} - diff --git a/virtual-banker/widget/src/components/VoiceControls.tsx b/virtual-banker/widget/src/components/VoiceControls.tsx deleted file mode 100644 index cbadbb7..0000000 --- a/virtual-banker/widget/src/components/VoiceControls.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import React, { useState } from 'react'; -import './VoiceControls.css'; - -interface VoiceControlsProps { - onPushToTalk: () => void; - onHandsFree: () => void; - isListening: boolean; - isMuted: boolean; - onToggleMute: () => void; - volume: number; - onVolumeChange: (volume: number) => void; -} - -export const VoiceControls: React.FC = ({ - onPushToTalk, - onHandsFree, - isListening, - isMuted, - onToggleMute, - volume, - onVolumeChange, -}) => { - const [mode, setMode] = useState<'push-to-talk' | 'hands-free'>('push-to-talk'); - - const handleModeChange = (newMode: 'push-to-talk' | 'hands-free') => { - setMode(newMode); - if (newMode === 'push-to-talk') { - onPushToTalk(); - } else { - onHandsFree(); - } - }; - - return ( -
-
- - -
- -
- - -
- - onVolumeChange(Number(e.target.value))} - className="volume-slider" - aria-label="Volume" - /> - {volume}% -
- - {isListening && ( - - 🎤 Listening - - )} -
-
- ); -}; - diff --git a/virtual-banker/widget/src/hooks/useConversation.ts b/virtual-banker/widget/src/hooks/useConversation.ts deleted file mode 100644 index 7442ae9..0000000 --- a/virtual-banker/widget/src/hooks/useConversation.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { useState, useCallback } from 'react'; -import { Message } from '../types'; - -export function useConversation() { - const [messages, setMessages] = useState([]); - const [isListening, setIsListening] = useState(false); - const [isSpeaking, setIsSpeaking] = useState(false); - - const addMessage = useCallback((message: Omit) => { - const newMessage: Message = { - ...message, - id: `msg-${Date.now()}-${Math.random()}`, - timestamp: new Date(), - }; - setMessages((prev) => [...prev, newMessage]); - }, []); - - const sendMessage = useCallback((content: string) => { - addMessage({ - role: 'user', - content, - }); - }, [addMessage]); - - const receiveMessage = useCallback((content: string) => { - addMessage({ - role: 'assistant', - content, - }); - }, [addMessage]); - - const clearMessages = useCallback(() => { - setMessages([]); - }, []); - - return { - messages, - isListening, - isSpeaking, - setIsListening, - setIsSpeaking, - sendMessage, - receiveMessage, - clearMessages, - }; -} - diff --git a/virtual-banker/widget/src/hooks/useSession.ts b/virtual-banker/widget/src/hooks/useSession.ts deleted file mode 100644 index 2d8fd32..0000000 --- a/virtual-banker/widget/src/hooks/useSession.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { useState, useEffect } from 'react'; -import { APIClient } from '../services/api'; -import { Session, WidgetConfig } from '../types'; - -export function useSession(config: WidgetConfig) { - const [session, setSession] = useState(null); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - - const apiClient = new APIClient(config); - - const createSession = async () => { - if (!config.tenantId || !config.userId) { - setError('tenantId and userId are required'); - return; - } - - setLoading(true); - setError(null); - - try { - const authAssertion = config.authToken || 'anonymous'; - const newSession = await apiClient.createSession( - config.tenantId, - config.userId, - authAssertion - ); - setSession(newSession); - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to create session'); - } finally { - setLoading(false); - } - }; - - const refreshToken = async () => { - if (!session) return; - - try { - const result = await apiClient.refreshToken(session.sessionId); - setSession({ - ...session, - ephemeralToken: result.ephemeralToken, - expiresAt: result.expiresAt, - }); - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to refresh token'); - } - }; - - const endSession = async () => { - if (!session) return; - - try { - await apiClient.endSession(session.sessionId); - setSession(null); - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to end session'); - } - }; - - // Auto-refresh token before expiration - useEffect(() => { - if (!session) return; - - const expiresAt = new Date(session.expiresAt).getTime(); - const now = Date.now(); - const timeUntilExpiry = expiresAt - now; - const refreshTime = timeUntilExpiry - 5 * 60 * 1000; // Refresh 5 minutes before expiry - - if (refreshTime > 0) { - const timer = setTimeout(() => { - refreshToken(); - }, refreshTime); - - return () => clearTimeout(timer); - } - }, [session]); - - return { - session, - loading, - error, - createSession, - refreshToken, - endSession, - }; -} - diff --git a/virtual-banker/widget/src/hooks/useWebRTC.ts b/virtual-banker/widget/src/hooks/useWebRTC.ts deleted file mode 100644 index f6a3760..0000000 --- a/virtual-banker/widget/src/hooks/useWebRTC.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { useState, useRef, useEffect } from 'react'; - -export function useWebRTC() { - const [isConnected, setIsConnected] = useState(false); - const [localStream, setLocalStream] = useState(null); - const [remoteStream, setRemoteStream] = useState(null); - const peerConnectionRef = useRef(null); - - const initializeWebRTC = async () => { - try { - // Get user media - const stream = await navigator.mediaDevices.getUserMedia({ - audio: true, - video: false, // Audio only for now - }); - setLocalStream(stream); - - // Create peer connection (simplified - should use proper signaling) - const pc = new RTCPeerConnection({ - iceServers: [ - { urls: 'stun:stun.l.google.com:19302' }, - ], - }); - - // Add local stream tracks - stream.getTracks().forEach((track) => { - pc.addTrack(track, stream); - }); - - // Handle remote stream - pc.ontrack = (event) => { - setRemoteStream(event.streams[0]); - }; - - pc.onconnectionstatechange = () => { - setIsConnected(pc.connectionState === 'connected'); - }; - - peerConnectionRef.current = pc; - } catch (err) { - console.error('Failed to initialize WebRTC:', err); - } - }; - - const closeWebRTC = () => { - if (localStream) { - localStream.getTracks().forEach((track) => track.stop()); - setLocalStream(null); - } - - if (peerConnectionRef.current) { - peerConnectionRef.current.close(); - peerConnectionRef.current = null; - } - - setRemoteStream(null); - setIsConnected(false); - }; - - useEffect(() => { - return () => { - closeWebRTC(); - }; - }, []); - - return { - isConnected, - localStream, - remoteStream, - initializeWebRTC, - closeWebRTC, - }; -} - diff --git a/virtual-banker/widget/src/index.css b/virtual-banker/widget/src/index.css deleted file mode 100644 index a38a246..0000000 --- a/virtual-banker/widget/src/index.css +++ /dev/null @@ -1,33 +0,0 @@ -* { - box-sizing: border-box; - margin: 0; - padding: 0; -} - -body { - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', - 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - -#virtual-banker-widget { - position: fixed; - bottom: 20px; - right: 20px; - z-index: 9999; -} - -/* Accessibility: Focus styles */ -*:focus { - outline: 2px solid #0066cc; - outline-offset: 2px; -} - -/* Keyboard navigation support */ -button:focus-visible, -input:focus-visible { - outline: 2px solid #0066cc; - outline-offset: 2px; -} - diff --git a/virtual-banker/widget/src/index.tsx b/virtual-banker/widget/src/index.tsx deleted file mode 100644 index c1d2475..0000000 --- a/virtual-banker/widget/src/index.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import React from 'react'; -import ReactDOM from 'react-dom/client'; -import { App } from './App'; -import './index.css'; - -// Initialize widget when DOM is ready -function initWidget() { - const containerId = 'virtual-banker-widget'; - let container = document.getElementById(containerId); - - if (!container) { - container = document.createElement('div'); - container.id = containerId; - document.body.appendChild(container); - } - - const root = ReactDOM.createRoot(container); - root.render( - - - - ); -} - -// Auto-initialize if script is loaded -if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', initWidget); -} else { - initWidget(); -} - -// Export for manual initialization -(window as any).VirtualBankerWidget = { - init: initWidget, -}; - diff --git a/virtual-banker/widget/src/services/api.ts b/virtual-banker/widget/src/services/api.ts deleted file mode 100644 index 842fc2d..0000000 --- a/virtual-banker/widget/src/services/api.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { Session, WidgetConfig } from '../types'; - -const DEFAULT_API_URL = 'http://localhost:8081'; - -export class APIClient { - private apiUrl: string; - private authToken?: string; - - constructor(config: WidgetConfig) { - this.apiUrl = config.apiUrl || DEFAULT_API_URL; - this.authToken = config.authToken; - } - - async createSession(tenantId: string, userId: string, authAssertion: string): Promise { - const response = await fetch(`${this.apiUrl}/v1/sessions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - ...(this.authToken && { Authorization: `Bearer ${this.authToken}` }), - }, - body: JSON.stringify({ - tenant_id: tenantId, - user_id: userId, - auth_assertion: authAssertion, - }), - }); - - if (!response.ok) { - const error = await response.json(); - throw new Error(error.message || 'Failed to create session'); - } - - return response.json(); - } - - async refreshToken(sessionId: string): Promise<{ ephemeralToken: string; expiresAt: string }> { - const response = await fetch(`${this.apiUrl}/v1/sessions/${sessionId}/refresh-token`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - ...(this.authToken && { Authorization: `Bearer ${this.authToken}` }), - }, - }); - - if (!response.ok) { - const error = await response.json(); - throw new Error(error.message || 'Failed to refresh token'); - } - - return response.json(); - } - - async endSession(sessionId: string): Promise { - const response = await fetch(`${this.apiUrl}/v1/sessions/${sessionId}/end`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - ...(this.authToken && { Authorization: `Bearer ${this.authToken}` }), - }, - }); - - if (!response.ok) { - const error = await response.json(); - throw new Error(error.message || 'Failed to end session'); - } - } -} - diff --git a/virtual-banker/widget/src/services/postMessage.ts b/virtual-banker/widget/src/services/postMessage.ts deleted file mode 100644 index d8ea0e7..0000000 --- a/virtual-banker/widget/src/services/postMessage.ts +++ /dev/null @@ -1,64 +0,0 @@ -export interface PostMessageEvent { - type: string; - payload?: any; -} - -export class PostMessageAPI { - private targetOrigin: string; - - constructor(targetOrigin: string = '*') { - this.targetOrigin = targetOrigin; - } - - // Send events to parent window - send(type: string, payload?: any): void { - if (typeof window !== 'undefined' && window.parent) { - window.parent.postMessage( - { - type, - payload, - source: 'virtual-banker-widget', - }, - this.targetOrigin - ); - } - } - - // Listen for messages from parent window - on(type: string, callback: (payload?: any) => void): () => void { - const handler = (event: MessageEvent) => { - if (event.data && event.data.type === type && event.data.source === 'virtual-banker-host') { - callback(event.data.payload); - } - }; - - window.addEventListener('message', handler); - - // Return unsubscribe function - return () => { - window.removeEventListener('message', handler); - }; - } - - // Widget events - ready(): void { - this.send('ready'); - } - - sessionStarted(sessionId: string): void { - this.send('session_started', { sessionId }); - } - - actionRequested(action: string, params: any): void { - this.send('action_requested', { action, params }); - } - - actionCompleted(action: string, result: any): void { - this.send('action_completed', { action, result }); - } - - handoffToHuman(reason: string): void { - this.send('handoff_to_human', { reason }); - } -} - diff --git a/virtual-banker/widget/src/types/index.ts b/virtual-banker/widget/src/types/index.ts deleted file mode 100644 index 1de71ed..0000000 --- a/virtual-banker/widget/src/types/index.ts +++ /dev/null @@ -1,55 +0,0 @@ -export interface WidgetConfig { - tenantId: string; - userId?: string; - authToken?: string; - apiUrl?: string; - theme?: ThemeConfig; - avatarEnabled?: boolean; - greeting?: string; -} - -export interface ThemeConfig { - primaryColor?: string; - secondaryColor?: string; - backgroundColor?: string; - textColor?: string; - logo?: string; -} - -export interface Session { - sessionId: string; - ephemeralToken: string; - config: TenantConfig; - expiresAt: string; -} - -export interface TenantConfig { - theme: ThemeConfig; - avatarEnabled: boolean; - greeting: string; - allowedTools: string[]; - policy: PolicyConfig; -} - -export interface PolicyConfig { - maxSessionDuration: number; - rateLimitPerMinute: number; - requireConsent: boolean; -} - -export interface Message { - id: string; - role: 'user' | 'assistant'; - content: string; - timestamp: Date; - audioUrl?: string; -} - -export interface ConversationState { - sessionId: string; - messages: Message[]; - isListening: boolean; - isSpeaking: boolean; - isConnected: boolean; -} - diff --git a/virtual-banker/widget/tsconfig.json b/virtual-banker/widget/tsconfig.json deleted file mode 100644 index b57c8d5..0000000 --- a/virtual-banker/widget/tsconfig.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2020", - "module": "ESNext", - "lib": ["ES2020", "DOM", "DOM.Iterable"], - "jsx": "react-jsx", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "moduleResolution": "node", - "resolveJsonModule": true, - "isolatedModules": true, - "noEmit": true, - "declaration": true, - "outDir": "./dist", - "baseUrl": ".", - "paths": { - "@/*": ["src/*"] - } - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} - diff --git a/virtual-banker/widget/webpack.config.js b/virtual-banker/widget/webpack.config.js deleted file mode 100644 index c259f2c..0000000 --- a/virtual-banker/widget/webpack.config.js +++ /dev/null @@ -1,45 +0,0 @@ -const path = require('path'); -const HtmlWebpackPlugin = require('html-webpack-plugin'); - -module.exports = { - entry: './src/index.tsx', - output: { - path: path.resolve(__dirname, 'dist'), - filename: 'widget.js', - library: 'VirtualBankerWidget', - libraryTarget: 'umd', - globalObject: 'this', - clean: true, - }, - resolve: { - extensions: ['.ts', '.tsx', '.js', '.jsx'], - alias: { - '@': path.resolve(__dirname, 'src'), - }, - }, - module: { - rules: [ - { - test: /\.tsx?$/, - use: 'ts-loader', - exclude: /node_modules/, - }, - { - test: /\.css$/, - use: ['style-loader', 'css-loader'], - }, - ], - }, - plugins: [ - new HtmlWebpackPlugin({ - template: './public/index.html', - filename: 'index.html', - }), - ], - externals: { - react: 'React', - 'react-dom': 'ReactDOM', - }, - devtool: 'source-map', -}; -