feat(agent): add web browsing and search capability

- Add WebExecutor with fetch_url and web_search tools using DuckDuckGo
- Include URL validation with proper subdomain matching for security
- Add web_browsing config option to enable/disable web tools
- Fix domain matching to prevent subdomain spoofing (e.g., untrusted.com
  no longer matches trusted.com in allowlist)
- Add urlencoding dependency for query encoding

Closes fluent_cli-a98

Author: njfio

crates/fluent-agent/Cargo.toml

@@ -44,14 +44,16 @@ prometheus = { workspace = true }
 # Additional utilities
 base64 = { workspace = true }
 thiserror = { workspace = true }
+urlencoding = "2.1"
 bincode = "1.3"
 toml = { workspace = true }
 # Web dashboard dependencies
 warp = "0.3"
 futures-util = { version = "0.3", features = ["sink", "std"] }
+# Async cancellation support
+tokio-util = "0.7"
 
 [dev-dependencies]
 tempfile = { workspace = true }
-tokio-util = "0.7"
 tokio-stream = "0.1"
 futures = { workspace = true }

crates/fluent-agent/src/config.rs

@@ -13,6 +13,62 @@ use crate::autonomy::AutonomySupervisorConfig;
 use crate::performance::PerformanceConfig;
 use crate::state_manager::StateManagerConfig;
 
+/// Rate limiting configuration for API calls
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RateLimitConfig {
+    /// Whether rate limiting is enabled
+    pub enabled: bool,
+    /// Maximum requests per second for reasoning engine
+    pub reasoning_rps: f64,
+    /// Maximum requests per second for action engine
+    pub action_rps: f64,
+    /// Maximum requests per second for reflection engine
+    pub reflection_rps: f64,
+}
+
+impl Default for RateLimitConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            reasoning_rps: 5.0,  // 5 requests per second
+            action_rps: 10.0,   // 10 requests per second
+            reflection_rps: 3.0, // 3 requests per second
+        }
+    }
+}
+
+impl RateLimitConfig {
+    /// Create from environment variables
+    pub fn from_environment() -> Self {
+        let enabled = std::env::var("FLUENT_RATE_LIMIT_ENABLED")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .unwrap_or(true);
+
+        let reasoning_rps = std::env::var("FLUENT_REASONING_RPS")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .unwrap_or(5.0);
+
+        let action_rps = std::env::var("FLUENT_ACTION_RPS")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .unwrap_or(10.0);
+
+        let reflection_rps = std::env::var("FLUENT_REFLECTION_RPS")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .unwrap_or(3.0);
+
+        Self {
+            enabled,
+            reasoning_rps,
+            action_rps,
+            reflection_rps,
+        }
+    }
+}
+
 /// Configuration for the agentic framework that integrates with fluent_cli's existing patterns
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AgentConfig {
@@ -33,6 +89,7 @@ pub struct AgentEngineConfig {
     pub supervisor: Option<AutonomySupervisorConfig>,
     pub performance: Option<PerformanceConfig>,
     pub state_management: Option<StateManagerConfig>,
+    pub rate_limit: Option<RateLimitConfig>,
 }
 
 /// Tool configuration for the agent
@@ -42,10 +99,16 @@ pub struct ToolConfig {
     pub shell_commands: bool,
     pub rust_compiler: bool,
     pub git_operations: bool,
+    #[serde(default = "default_web_browsing")]
+    pub web_browsing: bool,
     pub allowed_paths: Option<Vec<String>>,
     pub allowed_commands: Option<Vec<String>>,
 }
 
+fn default_web_browsing() -> bool {
+    true
+}
+
 /// Runtime configuration with loaded engines and credentials
 #[derive(Clone)]
 pub struct AgentRuntimeConfig {
@@ -57,6 +120,10 @@ pub struct AgentRuntimeConfig {
     pub supervisor: Option<AutonomySupervisorConfig>,
     pub performance: PerformanceConfig,
     pub state_overrides: Option<StateManagerConfig>,
+    pub rate_limit: RateLimitConfig,
+    pub reasoning_rate_limiter: Option<Arc<fluent_engines::RateLimiter>>,
+    pub action_rate_limiter: Option<Arc<fluent_engines::RateLimiter>>,
+    pub reflection_rate_limiter: Option<Arc<fluent_engines::RateLimiter>>,
 }
 
 impl AgentRuntimeConfig {
@@ -68,6 +135,96 @@ impl AgentRuntimeConfig {
         // In a real implementation, we'd need to restructure to avoid this type mismatch
         None
     }
+
+    /// Acquire a rate limit token for reasoning operations
+    ///
+    /// If rate limiting is disabled, returns immediately.
+    /// Otherwise, waits until a token is available.
+    pub async fn acquire_reasoning_rate_limit(&self) {
+        if let Some(ref limiter) = self.reasoning_rate_limiter {
+            limiter.acquire().await;
+        }
+    }
+
+    /// Acquire a rate limit token for action operations
+    pub async fn acquire_action_rate_limit(&self) {
+        if let Some(ref limiter) = self.action_rate_limiter {
+            limiter.acquire().await;
+        }
+    }
+
+    /// Acquire a rate limit token for reflection operations
+    pub async fn acquire_reflection_rate_limit(&self) {
+        if let Some(ref limiter) = self.reflection_rate_limiter {
+            limiter.acquire().await;
+        }
+    }
+
+    /// Try to acquire a rate limit token without blocking
+    ///
+    /// Returns true if token was acquired, false if rate limited.
+    pub async fn try_acquire_reasoning_rate_limit(&self) -> bool {
+        if let Some(ref limiter) = self.reasoning_rate_limiter {
+            limiter.try_acquire().await
+        } else {
+            true // No limiter = always allowed
+        }
+    }
+
+    /// Try to acquire an action rate limit token without blocking
+    pub async fn try_acquire_action_rate_limit(&self) -> bool {
+        if let Some(ref limiter) = self.action_rate_limiter {
+            limiter.try_acquire().await
+        } else {
+            true
+        }
+    }
+
+    /// Try to acquire a reflection rate limit token without blocking
+    pub async fn try_acquire_reflection_rate_limit(&self) -> bool {
+        if let Some(ref limiter) = self.reflection_rate_limiter {
+            limiter.try_acquire().await
+        } else {
+            true
+        }
+    }
+
+    /// Get the current rate limit configuration
+    pub fn rate_limit_config(&self) -> &RateLimitConfig {
+        &self.rate_limit
+    }
+
+    /// Check if rate limiting is enabled
+    pub fn is_rate_limiting_enabled(&self) -> bool {
+        self.rate_limit.enabled
+    }
+
+    /// Get available reasoning tokens (for monitoring)
+    pub async fn reasoning_tokens_available(&self) -> f64 {
+        if let Some(ref limiter) = self.reasoning_rate_limiter {
+            limiter.available_tokens().await
+        } else {
+            f64::INFINITY
+        }
+    }
+
+    /// Get available action tokens (for monitoring)
+    pub async fn action_tokens_available(&self) -> f64 {
+        if let Some(ref limiter) = self.action_rate_limiter {
+            limiter.available_tokens().await
+        } else {
+            f64::INFINITY
+        }
+    }
+
+    /// Get available reflection tokens (for monitoring)
+    pub async fn reflection_tokens_available(&self) -> f64 {
+        if let Some(ref limiter) = self.reflection_rate_limiter {
+            limiter.available_tokens().await
+        } else {
+            f64::INFINITY
+        }
+    }
 }
 
 impl AgentEngineConfig {
@@ -147,6 +304,20 @@ impl AgentEngineConfig {
             .await?
         };
 
+        // Initialize rate limiters based on config
+        let rate_limit_config = self.rate_limit.clone().unwrap_or_else(RateLimitConfig::from_environment);
+
+        let (reasoning_rate_limiter, action_rate_limiter, reflection_rate_limiter) =
+            if rate_limit_config.enabled {
+                (
+                    Some(Arc::new(fluent_engines::RateLimiter::new(rate_limit_config.reasoning_rps))),
+                    Some(Arc::new(fluent_engines::RateLimiter::new(rate_limit_config.action_rps))),
+                    Some(Arc::new(fluent_engines::RateLimiter::new(rate_limit_config.reflection_rps))),
+                )
+            } else {
+                (None, None, None)
+            };
+
         Ok(AgentRuntimeConfig {
             reasoning_engine: Arc::new(reasoning_engine),
             action_engine: Arc::new(action_engine),
@@ -156,6 +327,10 @@ impl AgentEngineConfig {
             supervisor: self.supervisor.clone(),
             performance: self.performance.clone().unwrap_or_default(),
             state_overrides: self.state_management.clone(),
+            rate_limit: rate_limit_config,
+            reasoning_rate_limiter,
+            action_rate_limiter,
+            reflection_rate_limiter,
         })
     }
 
@@ -362,6 +537,7 @@ impl AgentEngineConfig {
                 shell_commands: false, // Disabled by default for security
                 rust_compiler: true,
                 git_operations: false, // Disabled by default for security
+                web_browsing: true,
                 allowed_paths: Some(vec![
                     "./".to_string(),
                     "./src".to_string(),
@@ -381,6 +557,7 @@ impl AgentEngineConfig {
             supervisor: None,
             performance: None,
             state_management: None,
+            rate_limit: Some(RateLimitConfig::default()),
         }
     }
 
@@ -403,6 +580,7 @@ impl Default for ToolConfig {
             shell_commands: false,
             rust_compiler: true,
             git_operations: false,
+            web_browsing: true,
             allowed_paths: Some(vec![
                 "./".to_string(),
                 "./src".to_string(),
@@ -581,4 +759,36 @@ mod tests {
         let engines = vec!["sonnet3.5".to_string()];
         assert!(credentials::validate_credentials(&credentials, &engines).is_err());
     }
+
+    #[test]
+    fn test_rate_limit_config_default() {
+        let config = RateLimitConfig::default();
+        assert!(config.enabled);
+        assert_eq!(config.reasoning_rps, 5.0);
+        assert_eq!(config.action_rps, 10.0);
+        assert_eq!(config.reflection_rps, 3.0);
+    }
+
+    #[test]
+    fn test_rate_limit_config_from_env() {
+        // Test that environment variables are read correctly
+        std::env::set_var("FLUENT_RATE_LIMIT_ENABLED", "false");
+        std::env::set_var("FLUENT_REASONING_RPS", "2.5");
+
+        let config = RateLimitConfig::from_environment();
+        assert!(!config.enabled);
+        assert_eq!(config.reasoning_rps, 2.5);
+
+        // Clean up
+        std::env::remove_var("FLUENT_RATE_LIMIT_ENABLED");
+        std::env::remove_var("FLUENT_REASONING_RPS");
+    }
+
+    #[test]
+    fn test_default_config_includes_rate_limit() {
+        let config = AgentEngineConfig::default_config();
+        assert!(config.rate_limit.is_some());
+        let rate_limit = config.rate_limit.unwrap();
+        assert!(rate_limit.enabled);
+    }
 }