{"_id":"@lucid-spark/ddg-web-search","_rev":"3-587a072d96f9ffd6d415abb62716420a","name":"@lucid-spark/ddg-web-search","dist-tags":{"latest":"1.0.6"},"versions":{"1.0.3":{"name":"@lucid-spark/ddg-web-search","version":"1.0.3","keywords":["duckduckgo","search","web","fetcher","typescript","cli","command-line","api","rate-limiting","http-client","mcp","model-context-protocol","mcp-server"],"author":{"name":"DDG Web Search Contributors"},"license":"MIT","_id":"@lucid-spark/ddg-web-search@1.0.3","maintainers":[{"name":"lucivuc","email":"lucian.vuc@gmail.com"}],"homepage":"https://github.com/lucivuc/ddg-web-search#readme","bugs":{"url":"https://github.com/lucivuc/ddg-web-search/issues"},"bin":{"ddg-web-search":"dist/cli.js","ddg-web-search-mcp":"dist/mcp.js"},"dist":{"shasum":"1d5eb92e473f7f8cacf81168f09a1f1ffbd84d7e","tarball":"https://registry.npmjs.org/@lucid-spark/ddg-web-search/-/ddg-web-search-1.0.3.tgz","fileCount":51,"integrity":"sha512-bCDm7hXc25vsrkWZdl7I7TdkDJTmNIdmfOO4cMwPSP7A1/DrU33jsA7CR2+K+8oY4ETYGeK35GUO+Raj4hRwzQ==","signatures":[{"sig":"MEQCIFRveX4MJTE3ERF+qdIJX4QF+oakvQBYwWL0RhZf/y+2AiBoPSpc2sGCORxyTDV7XPJbbmn/bQQhv5twAecjKAxYhQ==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":114300},"main":"dist/index.js","types":"dist/index.d.ts","engines":{"npm":">=6.0.0","node":">=14.0.0"},"gitHead":"d31f464f75a88438b659b1315035805c7c722b15","scripts":{"cli":"npx ts-node src/cli.ts","dev":"npx ts-node examples/basic-usage.ts","mcp":"npx ts-node src/mcp.ts","test":"jest --coverage","build":"tsc","start":"node dist/index.js","format":"prettier --write","inspect":"npx -y @modelcontextprotocol/inspector npm run mcp","mcp:http":"npx ts-node src/mcp.ts --transport http","cli:build":"npm run build && node dist/cli.js","mcp:build":"npm run build && node dist/mcp.js","prepublish":"npm run build","inspect:http":"npx -y @modelcontextprotocol/inspector npm run mcp:http","test-package":"npx ts-node test-package.ts","format:staged":"pretty-quick --staged --no-restage --verbose","inspect:build":"npx -y @modelcontextprotocol/inspector npm run mcp:build","test-complete":"npx ts-node test-complete.ts","mcp:http:build":"npm run build && node dist/mcp.js --transport http","inspect:http:build":"npx -y @modelcontextprotocol/inspector npm run mcp:http:build"},"_npmUser":{"name":"lucivuc","email":"lucian.vuc@gmail.com"},"repository":{"url":"git+https://github.com/lucivuc/ddg-web-search.git","type":"git"},"_npmVersion":"10.9.2","description":"A comprehensive package for searching the web and fetching web content programmatically, through a CLI interface, or through a Model Context Protocol (MCP) server.","directories":{"doc":"docs","test":"tests","example":"examples"},"_nodeVersion":"22.16.0","dependencies":{"axios":"^1.6.0","cheerio":"^1.1.2","turndown":"^7.2.2","puppeteer":"^24.28.0","@types/puppeteer":"^5.4.7","@modelcontextprotocol/sdk":"^0.4.0"},"eslintConfig":{"extends":["plugin:@typescript-eslint/recommended"]},"_hasShrinkwrap":false,"devDependencies":{"jest":"^29.7.0","ts-jest":"^29.1.1","prettier":"^3.6.2","typescript":"^5.2.2","@types/jest":"^29.5.8","@types/node":"^20.8.10","pretty-quick":"^4.2.2","@types/cheerio":"^0.22.35","@types/turndown":"^5.0.6","@typescript-eslint/parser":"^6.12.0","@typescript-eslint/eslint-plugin":"^6.12.0"},"_npmOperationalInternal":{"tmp":"tmp/ddg-web-search_1.0.3_1763084154575_0.8258413782608047","host":"s3://npm-registry-packages-npm-production"}},"1.0.4":{"name":"@lucid-spark/ddg-web-search","version":"1.0.4","keywords":["duckduckgo","search","web","fetcher","typescript","cli","command-line","api","rate-limiting","http-client","mcp","model-context-protocol","mcp-server"],"author":{"name":"DDG Web Search Contributors"},"license":"MIT","_id":"@lucid-spark/ddg-web-search@1.0.4","maintainers":[{"name":"lucivuc","email":"lucian.vuc@gmail.com"}],"homepage":"https://github.com/lucivuc/ddg-web-search#readme","bugs":{"url":"https://github.com/lucivuc/ddg-web-search/issues"},"bin":{"ddg-web-search":"dist/cli.js","ddg-web-search-mcp":"dist/mcp.js"},"dist":{"shasum":"231c7cce44bbbea4cd629c572a674d56fff0ac7d","tarball":"https://registry.npmjs.org/@lucid-spark/ddg-web-search/-/ddg-web-search-1.0.4.tgz","fileCount":51,"integrity":"sha512-yqELkMwjmdh27ZB9Df5ZZDb8RpTxudNEPok6kATckvaWV1DP7enHyb6LuV6X1wvBfVm8Ay8BX1Y9EH7xNGrjIA==","signatures":[{"sig":"MEQCIBatPUcmlyVUeUZ+lLecnC4/hiQoFp301JN98IkB2lD5AiAV8vy/+0eNRdnsxkR6sjCv7j+OwCN4t6aEcdctMcSnuA==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":114300},"main":"dist/index.js","types":"dist/index.d.ts","engines":{"npm":">=6.0.0","node":">=14.0.0"},"gitHead":"db5e3348a8573e31e0073be58e202396569aa27e","scripts":{"cli":"npx ts-node src/cli.ts","dev":"npx ts-node examples/basic-usage.ts","mcp":"npx ts-node src/mcp.ts","test":"jest --coverage","build":"tsc","start":"node dist/index.js","format":"prettier --write","inspect":"npx -y @modelcontextprotocol/inspector npm run mcp","mcp:http":"npx ts-node src/mcp.ts --transport http","cli:build":"npm run build && node dist/cli.js","mcp:build":"npm run build && node dist/mcp.js","prepublish":"npm run build","inspect:http":"npx -y @modelcontextprotocol/inspector npm run mcp:http","test-package":"npx ts-node test-package.ts","format:staged":"pretty-quick --staged --no-restage --verbose","inspect:build":"npx -y @modelcontextprotocol/inspector npm run mcp:build","test-complete":"npx ts-node test-complete.ts","mcp:http:build":"npm run build && node dist/mcp.js --transport http","inspect:http:build":"npx -y @modelcontextprotocol/inspector npm run mcp:http:build"},"_npmUser":{"name":"lucivuc","email":"lucian.vuc@gmail.com"},"repository":{"url":"git+https://github.com/lucivuc/ddg-web-search.git","type":"git"},"_npmVersion":"10.9.2","description":"A comprehensive package for searching the web and fetching web content programmatically, through a CLI interface, or through a Model Context Protocol (MCP) server.","directories":{"doc":"docs","test":"tests","example":"examples"},"_nodeVersion":"22.16.0","dependencies":{"axios":"^1.6.0","cheerio":"^1.1.2","turndown":"^7.2.2","puppeteer":"^24.28.0","@types/puppeteer":"^5.4.7","@modelcontextprotocol/sdk":"^0.4.0"},"eslintConfig":{"extends":["plugin:@typescript-eslint/recommended"]},"_hasShrinkwrap":false,"devDependencies":{"jest":"^29.7.0","ts-jest":"^29.1.1","prettier":"^3.6.2","typescript":"^5.2.2","@types/jest":"^29.5.8","@types/node":"^20.8.10","pretty-quick":"^4.2.2","@types/cheerio":"^0.22.35","@types/turndown":"^5.0.6","@typescript-eslint/parser":"^6.12.0","@typescript-eslint/eslint-plugin":"^6.12.0"},"_npmOperationalInternal":{"tmp":"tmp/ddg-web-search_1.0.4_1763086555876_0.1755110364306134","host":"s3://npm-registry-packages-npm-production"}},"1.0.6":{"name":"@lucid-spark/ddg-web-search","version":"1.0.6","description":"A comprehensive package for searching the web and fetching web content programmatically, through a CLI interface, or through a Model Context Protocol (MCP) server.","main":"dist/index.js","types":"dist/index.d.ts","bin":{"ddg-web-search":"dist/cli.js","ddg-web-search-mcp":"dist/mcp.js"},"scripts":{"format":"prettier --write","format:staged":"pretty-quick --staged --no-restage --verbose","build":"tsc","test":"jest --coverage","start":"node dist/index.js","dev":"npx ts-node examples/basic-usage.ts","test-package":"npx ts-node test-package.ts","test-complete":"npx ts-node test-complete.ts","cli":"npx ts-node src/cli.ts","cli:build":"npm run build && node dist/cli.js","inspect":"npx -y @modelcontextprotocol/inspector npm run mcp","inspect:http":"npx -y @modelcontextprotocol/inspector npm run mcp:http","inspect:build":"npx -y @modelcontextprotocol/inspector npm run mcp:build","inspect:http:build":"npx -y @modelcontextprotocol/inspector npm run mcp:http:build","mcp":"npx ts-node src/mcp.ts","mcp:build":"npm run build && node dist/mcp.js","mcp:http":"npx ts-node src/mcp.ts --transport http","mcp:http:build":"npm run build && node dist/mcp.js --transport http","prepublish":"npm run build","docker:build":"docker build -t ddg-web-search .","docker:build:dev":"docker build -f Dockerfile.dev -t ddg-web-search:dev .","docker:run":"docker run -it ddg-web-search","docker:run:http":"docker run -p 3001:3001 ddg-web-search node dist/mcp.js --transport http --port 3001 --host 0.0.0.0","docker:run:cli":"docker run -it ddg-web-search node dist/cli.js interactive","docker:compose:http":"docker-compose --profile http up","docker:compose:stdio":"docker-compose --profile stdio up","docker:compose:cli":"docker-compose --profile cli up","docker:compose:dev":"docker-compose -f docker-compose.dev.yml up","docker:compose:test":"docker-compose -f docker-compose.dev.yml --profile test up ddg-test","docker:compose:down":"docker-compose down && docker-compose -f docker-compose.dev.yml down","docker:start":"./docker-start.sh","docker:test":"./docker-test.sh","docker:publish":"./docker-publish.sh","docker:publish:make":"make docker-publish","docker:publish:multiarch":"make docker-publish-multiarch"},"repository":{"type":"git","url":"git+https://github.com/lucivuc/ddg-web-search.git"},"keywords":["duckduckgo","ddg","search","web","fetcher","typescript","cli","command-line","api","rate-limiting","http-client","mcp","model-context-protocol","modelcontextprotocol","mcp-cli","mcp-server"],"author":{"name":"DDG Web Search Contributors"},"license":"MIT","homepage":"https://github.com/lucivuc/ddg-web-search#readme","bugs":{"url":"https://github.com/lucivuc/ddg-web-search/issues"},"engines":{"node":">=14.0.0","npm":">=6.0.0"},"dependencies":{"@modelcontextprotocol/sdk":"^0.4.0","@types/puppeteer":"^5.4.7","axios":"^1.6.0","cheerio":"^1.1.2","puppeteer":"^24.28.0","turndown":"^7.2.2"},"devDependencies":{"@types/cheerio":"^0.22.35","@types/jest":"^29.5.8","@types/node":"^20.8.10","@types/turndown":"^5.0.6","@typescript-eslint/eslint-plugin":"^6.12.0","@typescript-eslint/parser":"^6.12.0","jest":"^29.7.0","prettier":"^3.6.2","pretty-quick":"^4.2.2","ts-jest":"^29.1.1","typescript":"^5.2.2"},"eslintConfig":{"extends":["plugin:@typescript-eslint/recommended"]},"directories":{"doc":"docs","example":"examples","test":"tests"},"_id":"@lucid-spark/ddg-web-search@1.0.6","gitHead":"97c356aef2f833d9ff3042967c72fb1104e1ffaf","_nodeVersion":"22.20.0","_npmVersion":"10.9.3","dist":{"integrity":"sha512-dvQ9tp/TSbiMWBoByHO31Cq9iAxuQLxZ5WgD0qNbHRZCPZi2whpGkSjMH2rrMEUE6Ut0fD8+b26bu1/3AqDjOA==","shasum":"ac3143eaae45d69c5ab41de6fc2e5eb21f9550cf","tarball":"https://registry.npmjs.org/@lucid-spark/ddg-web-search/-/ddg-web-search-1.0.6.tgz","fileCount":51,"unpackedSize":121449,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIQDKna7NPi9FK+4UZNGCcFyAStlaKoOXnoWMungpyUJ1QgIgXCRYGNhEKg7lwdICFLuMN2RFW5dE3Ub9NScT5NgDgT4="}]},"_npmUser":{"name":"lucivuc","email":"lucian.vuc@gmail.com"},"maintainers":[{"name":"lucivuc","email":"lucian.vuc@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/ddg-web-search_1.0.6_1766832638672_0.455140678983053"},"_hasShrinkwrap":false}},"time":{"created":"2025-11-14T01:35:54.411Z","modified":"2025-12-27T10:50:39.044Z","1.0.3":"2025-11-14T01:35:54.759Z","1.0.4":"2025-11-14T02:15:56.085Z","1.0.6":"2025-12-27T10:50:38.824Z"},"bugs":{"url":"https://github.com/lucivuc/ddg-web-search/issues"},"author":{"name":"DDG Web Search Contributors"},"license":"MIT","homepage":"https://github.com/lucivuc/ddg-web-search#readme","keywords":["duckduckgo","ddg","search","web","fetcher","typescript","cli","command-line","api","rate-limiting","http-client","mcp","model-context-protocol","modelcontextprotocol","mcp-cli","mcp-server"],"repository":{"type":"git","url":"git+https://github.com/lucivuc/ddg-web-search.git"},"description":"A comprehensive package for searching the web and fetching web content programmatically, through a CLI interface, or through a Model Context Protocol (MCP) server.","maintainers":[{"name":"lucivuc","email":"lucian.vuc@gmail.com"}],"readme":"# DDG Web Search\n\nA comprehensive package for searching the web and fetching web content programmatically, through a CLI interface, or through a Model Context Protocol (MCP) server.\n\n## Features\n\n- **🔍 Web Searcher**: Search the web using DuckDuckGo with browser automation for reliable results\n- **📄 Web Content Fetcher**: Fetch and parse content from web pages with intelligent scraping\n- **⚡ MCP Server**: Model Context Protocol server with both stdio and HTTP (SSE) transport support\n- **💻 Command Line Interface**: Interactive CLI for searching and fetching content from the terminal\n- **🛡️ Rate Limiting**: Automatic request rate management to avoid hitting API limits\n- **🔧 Error Handling**: Robust error handling for browser automation, HTTP requests and network issues\n- **📝 TypeScript Support**: Full TypeScript support with comprehensive type definitions\n- **🤖 Browser Automation**: Uses Puppeteer for reliable web interaction and JavaScript rendering\n\n## Installation\n\n### NPM Installation\n\nTo install the package globally for CLI and MCP usage:\n\n```bash\nnpm install -g @lucid-spark/ddg-web-search\n```\n\nOr install locally in your project:\n\n```bash\nnpm install @lucid-spark/ddg-web-search\n```\n\nFor MCP server dependencies (automatically included):\n\n```bash\nnpm install @modelcontextprotocol/sdk puppeteer\n```\n\n### Docker Installation\n\nRun with Docker for easy deployment and consistent environment:\n\n```bash\n# Pull from Docker Hub\ndocker pull realLV/ddg-web-search:latest\n\n# Run HTTP server\ndocker run -p 3001:3001 realLV/ddg-web-search:latest \\\n  node dist/mcp.js --transport http --port 3001 --host 0.0.0.0\n\n# Run CLI interactively\ndocker run -it realLV/ddg-web-search:latest node dist/cli.js interactive\n```\n\nOr build locally:\n\n```bash\n# Build from source\ndocker build -t ddg-web-search .\n\n# Run HTTP server\ndocker run -p 3001:3001 ddg-web-search \\\n  node dist/mcp.js --transport http --port 3001 --host 0.0.0.0\n\n# Run CLI interactively\ndocker run -it ddg-web-search node dist/cli.js interactive\n```\n\nSee [DOCKER-HUB.md](./DOCKER-HUB.md) for publishing details and [DOCKER.md](./DOCKER.md) for comprehensive Docker usage guide.\n\n## Key Dependencies\n\n- **Puppeteer**: For reliable browser automation and JavaScript rendering\n- **Axios**: For HTTP requests in web content fetching\n- **Cheerio**: For HTML parsing and content extraction\n- **Turndown**: For converting HTML to Markdown\n- **MCP SDK**: For Model Context Protocol server functionality\n\n## MCP Server Usage\n\nThe package includes a Model Context Protocol (MCP) server that exposes the search and web content fetching as tools for AI assistants. The server supports both **stdio** and **HTTP** transports.\n\n### Available MCP Tools\n\n1. **search**: Search the web using DuckDuckGo with browser automation\n   - Input: `{ \"query\": \"search terms\" }`\n   - Returns: Formatted list of search results with titles, URLs, and snippets\n   - Features: Captcha handling, anti-detection measures, JavaScript rendering\n\n2. **fetch_web_content**: Fetch and parse content from a web URL with intelligent scraping\n   - Input: `{ \"url\": \"https://example.com\" }`\n   - Returns: Parsed text content with metadata (content truncated to 10,000 characters if needed)\n   - Features: HTML-to-Markdown conversion, metadata extraction, content cleaning\n\n### Running the MCP Server\n\n#### Stdio Transport (Default)\n\n```bash\n# Using global binary (after global install)\nddg-web-search-mcp\n\n# Using npx\nnpx @lucid-spark/ddg-web-search mcp\n\n# Using built files\nnpm run build\nnode dist/mcp.js\n```\n\n#### HTTP Transport\n\n```bash\n# Development mode with HTTP transport\nnpm run mcp:http\n\n# Production mode with HTTP transport\nnpm run mcp:http:build\n\n# Or manually with custom port/host\nnode dist/mcp.js --transport http --port 3001 --host localhost\n```\n\n### MCP Configuration\n\n#### For Stdio Transport (Default)\n\nAdd to your MCP client configuration (e.g., Claude Desktop):\n\n```json\n{\n  \"mcpServers\": {\n    \"ddg-web-search\": {\n      \"command\": \"ddg-web-search-mcp\",\n      \"args\": [],\n      \"env\": {}\n    }\n  }\n}\n```\n\nOr using npx:\n\n```json\n{\n  \"mcpServers\": {\n    \"ddg-web-search\": {\n      \"command\": \"npx\",\n      \"args\": [\"@lucid-spark/ddg-web-search\", \"mcp\"],\n      \"env\": {}\n    }\n  }\n}\n```\n\n#### For HTTP Transport\n\nStart the HTTP server and configure your MCP client to connect to the HTTP endpoint:\n\n```bash\n# Start HTTP server (default port 3001)\nnpm run mcp:http:build\n\n# Server will be available at:\n# - Server info: http://localhost:3001/\n# - SSE endpoint: http://localhost:3001/sse\n# - Message endpoint: http://localhost:3001/message/{sessionId}\n```\n\n#### Testing HTTP Transport\n\nA test HTML client is available at `examples/http-client.html` to demonstrate HTTP transport functionality. Start the HTTP server and open the HTML file in a browser to test the connection.\n\n#### Command Line Options\n\n```bash\n# Show help\nnode dist/mcp.js --help\n\n# Available options:\n# -t, --transport <type>    Transport type: 'stdio' or 'http' (default: stdio)\n# -p, --port <number>      Port for HTTP transport (default: 3001)\n# -h, --host <string>      Host for HTTP transport (default: localhost)\n```\n\n### HTTP Transport Endpoints\n\nWhen using HTTP transport, the server provides:\n\n- **Server Info**: `GET /` - Returns server information and available endpoints\n- **SSE Connection**: `GET /sse` - Establishes Server-Sent Events connection for receiving responses\n- **Message Sending**: `POST /message/{sessionId}` - Sends MCP requests to the server\n\nExample HTTP endpoints:\n\n```\nGET http://localhost:3001/         # Server info\nGET http://localhost:3001/sse      # SSE connection\nPOST http://localhost:3001/message/{sessionId}  # Send messages\n```\n\n## CLI Usage\n\nAfter global installation, you can use the CLI with these commands:\n\n### Basic Commands\n\n```bash\n# Search the web\nddg-web-search search \"TypeScript tutorials\"\n\n# Fetch content from a URL\nddg-web-search fetch https://example.com\n\n# Start interactive mode\nddg-web-search interactive\n\n# Start MCP server (stdio transport)\nddg-web-search mcp\n\n# Start MCP server (HTTP transport)\nddg-web-search mcp-http\n\n# Show help\nddg-web-search help\n\n# Show version\nddg-web-search version\n```\n\n### Interactive Mode\n\nStart interactive mode for a more engaging experience:\n\n```bash\nddg-web-search interactive\n```\n\nIn interactive mode, you can use these commands:\n\n- `search <query>` or `s <query>` - Search the web\n- `fetch <url>` or `f <url>` - Fetch web content\n- `mcp` - Start MCP server (stdio transport)\n- `mcp-http` - Start MCP server (HTTP transport)\n- `help` or `h` - Show help\n- `version` or `v` - Show version\n- `clear` or `cls` - Clear screen\n- `exit` or `quit` or `q` - Exit interactive mode\n\n### CLI Examples\n\n```bash\n# Search for JavaScript tutorials\nddg-web-search search \"JavaScript tutorials\"\n\n# Fetch content from a specific URL\nddg-web-search fetch https://httpbin.org/html\n\n# Start interactive session\nddg-web-search interactive\n\n# Start MCP server with stdio transport\nddg-web-search mcp\n\n# Start MCP server with HTTP transport\nddg-web-search mcp-http\n\n# Get help\nddg-web-search help\n```\n\n## Programmatic API Usage\n\n### Basic Example\n\nHere's a simple example of how to use the WebSearcher and WebContentFetcher:\n\n```typescript\nimport { WebSearcher, WebContentFetcher } from \"@lucid-spark/ddg-web-search\";\n\nasync function main() {\n  const searcher = new WebSearcher();\n  const fetcher = new WebContentFetcher();\n\n  try {\n    // Search the web (uses Puppeteer browser automation with DuckDuckGo)\n    const results = await searcher.search(\"TypeScript\");\n    console.log(\"Search Results:\", results);\n\n    // Fetch content from the first result (if available)\n    if (results.length > 0) {\n      const fetchResult = await fetcher.fetch(results[0].url);\n      if (fetchResult.success && fetchResult.data?.content) {\n        console.log(\"Content Length:\", fetchResult.data.content.length);\n        console.log(\"Page Title:\", fetchResult.data.metadata?.title);\n      }\n    }\n  } catch (error) {\n    console.error(\"Error:\", error);\n  } finally {\n    // Important: Close browser to free resources\n    await searcher.close();\n  }\n}\n\nmain();\n```\n\n### Using the MCP Server Programmatically\n\n```typescript\nimport { MCPServer } from \"@lucid-spark/ddg-web-search/dist/mcp\";\n\nasync function runMCPServer() {\n  const server = new MCPServer();\n  await server.run();\n  // Server will handle MCP requests via stdio\n}\n\nrunMCPServer();\n```\n\n## API Reference\n\n### WebSearcher\n\n#### Constructor\n\n```typescript\nnew WebSearcher(headless?: boolean);\n```\n\n- `headless` (optional): Whether to run browser in headless mode (default: `true`)\n  - Set to `false` for debugging or manual captcha solving\n\n#### Methods\n\n- `search(query: string): Promise<SearchResult[]>`: Searches DuckDuckGo using browser automation and returns an array of search results. Thread-safe with built-in initialization mutex to prevent race conditions.\n- `close(): Promise<void>`: Closes the browser instance and frees resources (important for memory management)\n\n### WebContentFetcher\n\n#### Constructor\n\n```typescript\nnew WebContentFetcher(rateLimit?: number, rateLimitInterval?: number);\n```\n\n- `rateLimit` (optional): Number of requests allowed per interval (default: 1)\n- `rateLimitInterval` (optional): Time interval in milliseconds (default: 1000)\n\n#### Methods\n\n- `fetch(url: string): Promise<FetchResult>`: Fetches and returns the content of the specified URL\n\n### CLI\n\n#### Constructor\n\n```typescript\nnew CLI();\n```\n\n#### Methods\n\n- `run(args: string[]): Promise<void>`: Run CLI with command line arguments\n- `search(query: string): Promise<void>`: Perform a search and display results\n- `fetch(url: string): Promise<void>`: Fetch content from URL and display\n- `interactive(): Promise<void>`: Start interactive mode\n\n### MCPServer\n\n#### Constructor\n\n```typescript\nnew MCPServer();\n```\n\n#### Methods\n\n- `run(): Promise<void>`: Start the MCP server listening on stdio\n\n### Types\n\n#### SearchResult\n\n```typescript\ninterface SearchResult {\n  title: string; // The title of the search result\n  url: string; // The URL of the search result\n  snippet: string; // A brief description or snippet\n  icon?: string; // Optional icon URL for the result\n}\n```\n\n#### FetchResult\n\n```typescript\ninterface FetchResult {\n  success: boolean; // Whether the fetch was successful\n  data?: WebContent; // Parsed web content (if available)\n  error?: string; // Error message (if failed)\n}\n```\n\n#### WebContent\n\n```typescript\ninterface WebContent {\n  content: string; // Main content of the page (HTML converted to Markdown)\n  metadata?: {\n    // Optional metadata extracted from the page\n    title?: string; // Page title\n    description?: string; // Page description\n    url?: string; // Page URL\n    author?: string; // Page author\n    publishDate?: string; // Publication date\n  };\n}\n```\n\n## Docker Support\n\nThis project includes comprehensive Docker support for both development and production environments.\n\n### Quick Start with Docker\n\n```bash\n# Using Docker Compose - HTTP transport\ndocker-compose --profile http up\n\n# Using Docker Compose - Development mode\ndocker-compose -f docker-compose.dev.yml up\n\n# Using Docker directly\ndocker build -t ddg-web-search .\ndocker run -p 3001:3001 ddg-web-search \\\n  node dist/mcp.js --transport http --port 3001 --host 0.0.0.0\n```\n\n### Docker Features\n\n- 🐳 **Multi-stage builds** for optimized image size\n- 🔒 **Non-root user** for enhanced security\n- 🚀 **Multiple profiles** (stdio, http, cli)\n- 🔄 **Hot-reload** support in development mode\n- 🏥 **Health checks** for monitoring\n- 📦 **Pre-configured Chromium** for Puppeteer\n\nFor detailed Docker documentation, including advanced usage, troubleshooting, and best practices, see [DOCKER.md](./DOCKER.md).\n\n## Development\n\n### Configuration Constants\n\nThe package uses the following configurable constants:\n\n**WebSearcher**:\n\n- `RATE_LIMIT_INTERVAL_MS`: 2000ms (1 request per 2 seconds)\n- `NAVIGATION_TIMEOUT_MS`: 30000ms (30 seconds for page navigation)\n- `SELECTOR_WAIT_TIMEOUT_MS`: 5000ms (5 seconds for selector waiting)\n- `CAPTCHA_MANUAL_SOLVE_TIMEOUT_MS`: 60000ms (60 seconds for manual captcha solving)\n\n**HttpClient**:\n\n- `DEFAULT_TIMEOUT_MS`: 10000ms (10 seconds for HTTP requests)\n\n**MCP Server**:\n\n- `MAX_CONTENT_LENGTH`: 10000 characters (content truncation limit)\n- `DEFAULT_HTTP_PORT`: 3001 (HTTP transport default port)\n\n**CLI**:\n\n- `CONTENT_PREVIEW_LENGTH`: 500 characters (preview display limit)\n\n### Building the Project\n\n```bash\nnpm run build\n```\n\n### Running Tests\n\n```bash\nnpm test\n```\n\n### Development Scripts\n\n```bash\n# Run example usage\nnpm run dev\n\n# Test the package functionality\nnpm run test-package\n\n# Run CLI in development mode\nnpm run cli help\n\n# Run MCP server with stdio transport (development mode)\nnpm run mcp\n\n# Run MCP server with HTTP transport (development mode)\nnpm run mcp:http\n\n# Build and run compiled CLI\nnpm run cli:build help\n\n# Build and run compiled MCP server with stdio transport\nnpm run mcp:build\n\n# Build and run compiled MCP server with HTTP transport\nnpm run mcp:http:build\n```\n\n## Rate Limiting\n\nBoth the WebSearcher and WebContentFetcher include built-in rate limiting to prevent overwhelming servers:\n\n- **WebSearcher**: 1 request per 2 seconds (2000ms) by default - conservative timing for browser automation\n- **WebContentFetcher**: Configurable rate limiting (1 request per 1000ms by default)\n\n## Error Handling\n\nThe package includes comprehensive error handling:\n\n- Network errors are caught and logged\n- Invalid URLs are handled gracefully\n- Rate limiting is enforced automatically\n- HTTP errors are properly typed and handled\n- MCP server errors are returned as structured error responses\n- Browser automation errors (timeouts, captcha detection, connection issues) are handled gracefully\n- Captcha detection: Returns empty results in headless mode; waits up to 60 seconds for manual solving in non-headless mode\n- Resource cleanup ensures no memory leaks from browser instances\n- Thread-safe browser initialization prevents race conditions\n\n**Important Note**: Always call `searcher.close()` when done with WebSearcher to properly cleanup browser resources and prevent memory leaks.\n\n## CLI Features\n\nThe CLI provides a rich, colorful interface with:\n\n- 🎨 **Colorful Output**: Different colors for different types of messages\n- 🔍 **Search Functionality**: Search the web directly from the terminal using browser automation\n- 📥 **Content Fetching**: Fetch and preview web content with intelligent parsing\n- 🎯 **Interactive Mode**: Engaging interactive session with command history\n- 📖 **Comprehensive Help**: Built-in help system with examples\n- ⚡ **Fast Performance**: Built with TypeScript for optimal performance\n- 🧹 **Resource Management**: Automatic cleanup of browser resources\n\n## MCP Integration\n\nThe Model Context Protocol server enables AI assistants to:\n\n- 🔍 **Search the Web**: Use the web with advanced browser automation for JavaScript-rendered content\n- 📄 **Fetch Content**: Retrieve and parse content from web pages with intelligent scraping\n- 🚀 **Dual Transport Support**: Supports both stdio and HTTP (SSE) transports\n- ⚡ **Rate Limited**: Automatic rate limiting prevents API abuse\n- 🛡️ **Error Handling**: Graceful error handling with structured responses\n- 📡 **Standards Compliant**: Follows MCP specification for AI assistant integration\n- 🤖 **Captcha Aware**: Handles captcha challenges intelligently\n- 🧹 **Resource Efficient**: Proper cleanup of browser instances\n\n## Contributing\n\nContributions are welcome! Please open an issue or submit a pull request for any improvements or bug fixes.\n\n### Development Setup\n\n1. Clone the repository\n2. Install dependencies: `npm install`\n3. Build the project: `npm run build`\n4. Run tests: `npm test`\n5. Test CLI: `npm run cli help`\n6. Test MCP server: `npm run mcp`\n\n## License\n\nThis project is licensed under the MIT License. See the LICENSE file for details.\n","readmeFilename":"README.md"}