{"_id":"@danger-dream/web-crawler-mcp","_rev":"4-aef3121d628a109923148df5dcefb567","name":"@danger-dream/web-crawler-mcp","dist-tags":{"latest":"1.0.3"},"versions":{"1.0.0":{"name":"@danger-dream/web-crawler-mcp","version":"1.0.0","keywords":["mcp","model-context-protocol","web-crawler","web-search","ai"],"author":{"name":"danger-dream"},"license":"MIT","_id":"@danger-dream/web-crawler-mcp@1.0.0","maintainers":[{"name":"exaggerated-dream","email":"18213053121@163.com"}],"homepage":"https://github.com/danger-dream/web-crawler-mcp","bugs":{"url":"https://github.com/danger-dream/web-crawler-mcp/issues"},"bin":{"mcp-server-web-crawler":"dist/index.js"},"dist":{"shasum":"64ca7500b25a85a1f0c9af7d1d25d22ebdae577c","tarball":"https://registry.npmjs.org/@danger-dream/web-crawler-mcp/-/web-crawler-mcp-1.0.0.tgz","fileCount":17,"integrity":"sha512-IFurv4q0dc2Ayg09DDEvHoECm7fxyAy9Fl75lPj848awDU1MO8/PihBrloW2isCIqoDHzuS+CIqmkMG+Fki87A==","signatures":[{"sig":"MEQCIEw1woQ0+NEBqRG0nfGC96ogzq2uGD06FPBa6YO6BUKOAiB4fzA+t1ScZsk3z9RWE325sgOxC0ZQgOt2KGXCrDHGtQ==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":26186},"main":"dist/index.js","type":"module","scripts":{"build":"tsc","start":"node dist/index.js"},"_npmUser":{"name":"exaggerated-dream","email":"18213053121@163.com"},"_npmVersion":"10.2.4","description":"Web Crawler MCP","directories":{},"_nodeVersion":"21.6.2","dependencies":{"zod":"^3.24.2","happy-dom":"^17.4.4","node-html-markdown":"^1.3.0","zod-to-json-schema":"^3.24.4","@mozilla/readability":"^0.6.0","@modelcontextprotocol/sdk":"^1.7.0"},"_hasShrinkwrap":false,"devDependencies":{"ts-node":"^10.9.2","typescript":"5.8.2","@types/node":"^22.13.10"},"_npmOperationalInternal":{"tmp":"tmp/web-crawler-mcp_1.0.0_1742538737419_0.6443058331190223","host":"s3://npm-registry-packages-npm-production"}},"1.0.1":{"name":"@danger-dream/web-crawler-mcp","version":"1.0.1","keywords":["mcp","model-context-protocol","web-crawler","web-search","ai"],"author":{"name":"danger-dream"},"license":"MIT","_id":"@danger-dream/web-crawler-mcp@1.0.1","maintainers":[{"name":"exaggerated-dream","email":"18213053121@163.com"}],"homepage":"https://github.com/danger-dream/web-crawler-mcp","bugs":{"url":"https://github.com/danger-dream/web-crawler-mcp/issues"},"bin":{"mcp-server-web-crawler":"dist/index.js"},"dist":{"shasum":"9cd873acd456777677b722e99f7e511badf97ac1","tarball":"https://registry.npmjs.org/@danger-dream/web-crawler-mcp/-/web-crawler-mcp-1.0.1.tgz","fileCount":17,"integrity":"sha512-uE9SihU1jovSoprZqBcIke9Uj0N+QRaIYwhyjujoeC+y/nWbT4xo8oDJgnc45iC56I5M3jMCOmunSgV7S3c+UA==","signatures":[{"sig":"MEQCIFPcSBeFRqFWJJVvlzxn+koFKqcG+V5uqfecPIOfZGhBAiBwBdQIfBttnRDqYTf9lUHmiM50GIFn/ps3e59x8tDmpw==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":23130},"main":"dist/index.js","type":"module","scripts":{"build":"tsc","start":"node dist/index.js"},"_npmUser":{"name":"exaggerated-dream","email":"18213053121@163.com"},"_npmVersion":"10.2.4","description":"Web Crawler MCP","directories":{},"_nodeVersion":"21.6.2","dependencies":{"zod":"^3.24.2","happy-dom":"^17.4.4","node-html-markdown":"^1.3.0","zod-to-json-schema":"^3.24.4","@mozilla/readability":"^0.6.0","@modelcontextprotocol/sdk":"^1.7.0"},"_hasShrinkwrap":false,"devDependencies":{"ts-node":"^10.9.2","typescript":"5.8.2","@types/node":"^22.13.10"},"_npmOperationalInternal":{"tmp":"tmp/web-crawler-mcp_1.0.1_1742539333646_0.43143456163846206","host":"s3://npm-registry-packages-npm-production"}},"1.0.2":{"name":"@danger-dream/web-crawler-mcp","version":"1.0.2","keywords":["mcp","model-context-protocol","web-crawler","web-search","ai"],"author":{"name":"danger-dream"},"license":"MIT","_id":"@danger-dream/web-crawler-mcp@1.0.2","maintainers":[{"name":"exaggerated-dream","email":"18213053121@163.com"}],"homepage":"https://github.com/danger-dream/web-crawler-mcp","bugs":{"url":"https://github.com/danger-dream/web-crawler-mcp/issues"},"bin":{"mcp-server-web-crawler":"dist/index.js"},"dist":{"shasum":"09a17f3c5d9a6c449e2e79a2f1f9e3045de7d7fb","tarball":"https://registry.npmjs.org/@danger-dream/web-crawler-mcp/-/web-crawler-mcp-1.0.2.tgz","fileCount":17,"integrity":"sha512-1aYxU6cLLpkOr8iC9WtIxHbK6HX071Bkf+pA5/qETkt7VETFTWkBlMaCqZt1kBJYXNd1n9bvFLdCFIb6+ayTuA==","signatures":[{"sig":"MEUCIDaNYAsna0Msno2zDiTOHvfhSu/W7g+KX+JBN09dBsIiAiEAox/w87rSa/4kHgslCnAZGNj8B+z4sfxwx5udTdXDceE=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":23059},"main":"dist/index.js","type":"module","scripts":{"build":"tsc","start":"node dist/index.js","start:dev":"tsx src/index.ts"},"_npmUser":{"name":"exaggerated-dream","email":"18213053121@163.com"},"_npmVersion":"10.2.4","description":"Web Crawler MCP","directories":{},"_nodeVersion":"21.6.2","dependencies":{"zod":"^3.24.2","happy-dom":"^17.4.4","node-html-markdown":"^1.3.0","zod-to-json-schema":"^3.24.4","@mozilla/readability":"^0.6.0","@modelcontextprotocol/sdk":"^1.7.0"},"_hasShrinkwrap":false,"devDependencies":{"tsx":"^4.19.3","ts-node":"^10.9.2","typescript":"5.8.2","@types/node":"^22.13.10"},"_npmOperationalInternal":{"tmp":"tmp/web-crawler-mcp_1.0.2_1742540007910_0.09378878002346869","host":"s3://npm-registry-packages-npm-production"}},"1.0.3":{"name":"@danger-dream/web-crawler-mcp","version":"1.0.3","description":"Web Crawler MCP","license":"MIT","author":{"name":"danger-dream"},"homepage":"https://github.com/danger-dream/web-crawler-mcp","bugs":{"url":"https://github.com/danger-dream/web-crawler-mcp/issues"},"type":"module","bin":{"mcp-server-web-crawler":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"tsc","start":"node dist/index.js","start:dev":"tsx src/index.ts"},"keywords":["mcp","model-context-protocol","web-crawler","web-search","ai"],"dependencies":{"@modelcontextprotocol/sdk":"^1.7.0","@mozilla/readability":"^0.6.0","happy-dom":"^17.4.4","node-html-markdown":"^1.3.0","zod":"^3.24.2","zod-to-json-schema":"^3.24.4"},"devDependencies":{"@types/node":"^22.13.10","ts-node":"^10.9.2","tsx":"^4.19.3","typescript":"5.8.2"},"_id":"@danger-dream/web-crawler-mcp@1.0.3","_nodeVersion":"21.6.2","_npmVersion":"10.2.4","dist":{"integrity":"sha512-l439Ub75F5gI3ONh8PsX38dWvdM3tm1QgOlD0EJXloP4S8/fipg7z0bfOCFgY1yzEtP74NK1ifk79szRYD6Y0A==","shasum":"2f947815726defec5f41d5c2d7b33edc9eeda46c","tarball":"https://registry.npmjs.org/@danger-dream/web-crawler-mcp/-/web-crawler-mcp-1.0.3.tgz","fileCount":17,"unpackedSize":23098,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIDEYN4KOugxHlPd+n9zAj5/aqHRNOnLUP2KDz5qsdFnCAiEA5gB2xZ29VoJim2LUVc0nxP2Ky435iOuCYrBzVSblKsA="}]},"_npmUser":{"name":"exaggerated-dream","email":"18213053121@163.com"},"directories":{},"maintainers":[{"name":"exaggerated-dream","email":"18213053121@163.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/web-crawler-mcp_1.0.3_1742547638603_0.10962909261640363"},"_hasShrinkwrap":false}},"time":{"created":"2025-03-21T06:32:17.355Z","modified":"2025-03-21T09:00:38.954Z","1.0.0":"2025-03-21T06:32:17.616Z","1.0.1":"2025-03-21T06:42:13.827Z","1.0.2":"2025-03-21T06:53:28.077Z","1.0.3":"2025-03-21T09:00:38.761Z"},"bugs":{"url":"https://github.com/danger-dream/web-crawler-mcp/issues"},"author":{"name":"danger-dream"},"license":"MIT","homepage":"https://github.com/danger-dream/web-crawler-mcp","keywords":["mcp","model-context-protocol","web-crawler","web-search","ai"],"description":"Web Crawler MCP","maintainers":[{"name":"exaggerated-dream","email":"18213053121@163.com"}],"readme":"# Web Crawler MCP Service\n\nThe web-crawler of [LobeChat](https://github.com/lobehub/lobe-chat/tree/main/packages/web-crawler) is very useful, and it has been extracted to create MCP.\n\n## Features\n\n### Tools\n\n#### `searchWithSearXNG`\n\n- Provides powerful search functionality through the SearXNG meta search engine\n- Supports multiple search engines: Google, Bing, DuckDuckGo, Bilibili, etc.\n- Returns structured search results\n- Customizable search engine selection\n\n#### `crawlSinglePage`\n\n- Extracts content from web pages, optimized for LLM consumption\n- Multiple web content retrieval methods for reliability\n- Automatically extracts webpage titles and main content\n- Intelligent error handling and failover mechanisms\n\n#### `crawlMultiPages`\n\n- Crawls multiple web pages simultaneously\n- Parallel processing for improved efficiency\n- Shares the same features as single page crawling\n- Returns merged structured data\n\n## Crawling Implementation\n\nThe service uses multiple crawling methods, tried in priority order:\n\n1. **Naive**: Basic crawling implementation, directly fetches web page content\n2. **Jina**: Uses Jina AI's web reader API\n3. **Search1API**: Uses the Search1API service\n4. **Browserless**: Uses the Browserless.io service for browser rendering\n\n## Setup\n\n### Prerequisites\n\nYou'll need API keys for the following services to fully utilize this service:\n\n- SearXNG search engine instance\n- Jina AI API key (optional)\n- Search1API key (optional)\n- Browserless token (optional)\n\n### Installation\n\n#### Method 1: NPX (Recommended)\n\n```bash\nnpx -y @danger-dream/web-crawler-mcp\n```\n\nconfiguration:\n\n```json\n{\n  \"mcpServers\": {\n    \"deepsearch\": {\n      \"command\": \"npx\",\n      \"args\": [\"-y\", \"@danger-dream/web-crawler-mcp\"],\n      \"env\": {\n        \"SEARXNG_BASE_URL\": \"<Your SearXNG Instance URL>\",\n        \"JINA_READER_API_KEY\": \"<Your JINA Key>\",\n        \"BROWSERLESS_TOKEN\": \"<Your BROWSERLESS Token>\",\n        \"SEARCH1API_API_KEY\": \"<Your SEARCH1API Key>\"\n      }\n    }\n  }\n}\n```\n\n### Environment Variables\n\nThe service supports the following environment variables or command line parameters:\n\n- `SEARXNG_BASE_URL`: SearXNG search engine base URL (default: `http://localhost:8080`)\n- `JINA_READER_API_KEY`: Jina Reader API key\n- `BROWSERLESS_URL`: Browserless service URL (default: `https://chrome.browserless.io`)\n- `BROWSERLESS_TOKEN`: Browserless service token\n- `SEARCH1API_API_KEY`: Search1API service API key\n\n## License\n\nMIT\n","readmeFilename":"README.md"}