{"_id":"robotstxt","_rev":"37-dec47dbf7204431e103d117402b5676e","name":"robotstxt","description":"a robotstxt parser for node.js","dist-tags":{"latest":"0.0.4-7"},"versions":{"0.0.1-0":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.1-0","devDependencies":{},"_id":"robotstxt@0.0.1-0","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.6","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"3eae82ccae1b8a7e3e7f32797b79761185e55b6b","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.1-0.tgz","integrity":"sha512-TbF6ja7+2Y2EGBB9zYRQWmu0F4ytAiTsD4Yy+rQ5MgZ8oM3nMgXku9B+5XOAx2k7lvnpQd+MJ0+UvGWlJ1KKuA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDftQeYztTdSXF/pdYd0KLKiH1ojpNl2odZNwtVHnNk6wIgdivordH2QRTW80P83ZzetUmCZqS5EK1Th9MLlwIiWHI="}]},"scripts":{}},"0.0.1-4":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.1-4","devDependencies":{},"_id":"robotstxt@0.0.1-4","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"569a0f00492e4abea75afdb73be8a4b3ea11d0d8","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.1-4.tgz","integrity":"sha512-vFrQdw2UX/IaX3tj1edXpIhhwQZA+ucrXFIrC1C4EevCSeh3dbR5Pu9ICmrEaHrv+Ju0jOBUB4uFBG1XuoAu+A==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDvhsts7tUJiHCPS5JiE8zMieQ+d2rBd5K8wMSzMJOj7QIgI7kqrYMxtxAGaXMvS4kLzzEfWKOloXbzA8iA+OyBCnk="}]},"scripts":{}},"0.0.1-5":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.1-5","devDependencies":{},"_id":"robotstxt@0.0.1-5","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"00e3f82475a3133b12c73b7c3ff4098a9ab6c2fb","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.1-5.tgz","integrity":"sha512-prOIsVD00TfhRvV4s2OnEH1KsgTnB6lo3i6osiVdQdRiIj46k4SljZ5AZbN1nY0hHqjTUbDwEitxdA8/KKVymw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDE5T47vWsn1qNe5g5lz2NkvZO27ev2JvKWRE+6Mbj7FAIhAOXRwFuIfDaIereNshIHfrOkbXQ2P7bZOBlMEWjat4Xt"}]},"scripts":{}},"0.0.1-6":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.1-6","devDependencies":{},"_id":"robotstxt@0.0.1-6","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"420fd4594046f55365784a86c407577810f60ba2","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.1-6.tgz","integrity":"sha512-SovH7RNfN5nTzV4PQb2/eb3iIESfI/G+G0whSqysgJe55To4ens6AjHMGtldqgVeHex0pCnFKPUwlOVY3cBNFA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCQb/fWp8yZX2cOY2J3Yr1jWayi/Vk3UIW2KeWmX+JWnwIgIabQPsvZoxVbkD2RHxw5Btj4yhGznqxl2xWJ15ivub8="}]},"scripts":{}},"0.0.1-7":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.1-7","devDependencies":{},"_id":"robotstxt@0.0.1-7","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"08a5a2f4af1df8db198ab1bc32ffc449f2a80efd","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.1-7.tgz","integrity":"sha512-qxK60gqkfBfzlq67C4uRO3FpEuvyHOPbDaaZyMDj6f/lQF+zYevc6zfN1y3OK5nGy5OWvoqQD6sWrPPNtrLB4g==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIH3uFzIKz73n0Mq+g2w8khWAl+ub8rS5Yll/Y3Iu/a7cAiEArZzhoPWIDam6aVbzN6lsI/IzH9NMCs5T/QKKzcaG+EI="}]},"scripts":{}},"0.0.1-8":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.1-8","devDependencies":{},"_id":"robotstxt@0.0.1-8","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"b814ce0571ca36550dc985ccfe1736edc84f736a","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.1-8.tgz","integrity":"sha512-q/lZ0T3Tu7ZHpnOV4OoPyHt2zYWI4nZdgwMzSoJPd9ddfjQXDgrEjUCrc8IDAcSfH27xqmv8xYZ2Juck7LDRHQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIH1HZ22GOrb7iqXlMaUVURkyQ8ksZpj6euthLLLQNGEDAiEA1gWnpQAJTtG0TeqLzAZgBk8IY+f8cjzgBK7sofCTeDY="}]},"scripts":{}},"0.0.2-0":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.2-0","devDependencies":{},"_id":"robotstxt@0.0.2-0","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"b1273fb464fcdccbbe3ba8421b8e1d6a185ff0b2","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.2-0.tgz","integrity":"sha512-9GB1kR1uJgRspHAkyLWBigY3uzSCveU5ujEqT79Tfj3R5FCQJwNSkP8eUPBUX8j04BKQl69aSe8n77L8Yoss5Q==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDriDJRW7Ey18yQSmsiCn+paJSWOzTkpnwlqPKbMTLlCQIgaqrOxlK1cCU4SHoIm51BgT3RZi9zSbPAKuhEIxi8IOs="}]},"scripts":{}},"0.0.2-1":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.2-1","devDependencies":{},"_id":"robotstxt@0.0.2-1","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"5f131f41d57569fb9260140e8c0b9093d3f47ea8","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.2-1.tgz","integrity":"sha512-V6GzY4WB0Y1VBw1NfnsFxowsZDUT40TpK6e1PqY0vY/EPZQHy3AA5wPy19pGut66ld4T5fwBiIrEShH8398SvQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIC4D0UvGcMwILIxxSaftEX2/cQEmMTBYxTnvd+39cxNZAiBapQivf/N23orpkySLGSLwLgU/eRO3QqjpOxo28lmxBQ=="}]},"scripts":{}},"0.0.3-0":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.3-0","devDependencies":{},"_id":"robotstxt@0.0.3-0","engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.1","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"2b9332089f0928793eea37eddee9b00b6b30aacb","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.3-0.tgz","integrity":"sha512-wPTZxrB8kJ4t9wJE52ttrR05IKwhTO8fGb+hcUbNyunfmbr9Kxue8sBqFGRwNQQnrUdg9bLDUIJufnagCGLWjQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIEjFmKgpgy9nYawfaUhUt/X+zv+4KqsGAO68O1OjtXqrAiEAjBCAZGlY2jE6LMgswoC5vS7nN1sMWnoCiJFkVkfp/oc="}]},"scripts":{}},"0.0.3-1":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.3-1","_npmJsonOpts":{"file":"/Users/franzseo/.npm/robotstxt/0.0.3-1/package/package.json","wscript":false,"contributors":false,"serverjs":false},"_id":"robotstxt@0.0.3-1","devDependencies":{},"engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.28","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"02c1fb88c7926d28b9acfc514b9f07d805a0ea7a","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.3-1.tgz","integrity":"sha512-TKQqvFKaYckFK9flaWGyoNSRt/+C/ERYHDDUcS2cUdMT8hOjPTqyy4DisyUBvhkrvC3HFOOD0QTOD1Gxeba4IQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIEp8NtN0mW0SwCJOxWGKLh/6PjKeXmsGUN+JlnildQprAiBSa2EYnOKvcjuIQBKB23CZ+YCmc2J1pKkI/2GNFv4yGw=="}]},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]},"0.0.4-0":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.4-0","_npmJsonOpts":{"file":"/Users/franzseo/.npm/robotstxt/0.0.4-0/package/package.json","wscript":false,"contributors":false,"serverjs":false},"_id":"robotstxt@0.0.4-0","devDependencies":{},"engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.28","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"bf9b3c7b638ca4adc4d2345bbf84e55c4f0a529b","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.4-0.tgz","integrity":"sha512-vwn7mX+utgBFHj3A6uWanE9jz0TCu3THAVQSnEXXP6hKW1S5Z7S6f6HhVXR3bZxhChnlDQAlHFaDfVCtUZhh6g==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIFTJlAzK21AFRYGz1mmVSiLD8MDmPiuqiL2XL3Cth/kAAiEAsLDxK27KJLXXf4bVV1EHOSArjm/12fFQaMmtWXTkb8g="}]},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]},"0.0.4-1":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.4-1","_npmJsonOpts":{"file":"/Users/franzseo/.npm/robotstxt/0.0.4-1/package/package.json","wscript":false,"contributors":false,"serverjs":false},"_id":"robotstxt@0.0.4-1","devDependencies":{},"engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.28","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"1baa01a5e9c7d136df7ce5fe673056768458074f","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.4-1.tgz","integrity":"sha512-RJgWQ5bSsShvbeB2Q4w5Kc1ztWLuoFBPIYsBVoe7GzSJO+SvRFHhz18tagtZ4vYDeQjJUk4O1V8YBKIupU1E1g==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIEuqnMG+W5foH6SLW1BAZuSSyBcerHhQPTCQtCl6UIWwAiBzGrB4DuaUh3y1mFLoQWC9JhqUGcRhWB7EVNf+sXNSdA=="}]},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]},"0.0.4-3":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.4-3","_npmJsonOpts":{"file":"/Users/franzseo/.npm/robotstxt/0.0.4-3/package/package.json","wscript":false,"contributors":false,"serverjs":false},"_id":"robotstxt@0.0.4-3","devDependencies":{},"engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.28","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"fcdc089c74d8d10266822388d7a4fafb50cd836a","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.4-3.tgz","integrity":"sha512-Yhr4boKqocP184JzHHIO6e8F7zu8Pq4J9tXqdoBy3L9Z5f/rrtPhh4F1xdtH2Q3GFwtSu/RAL1hzAxYA6Z5k5Q==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCsLs0aKAV9UBDLcc/m957XQIXzjawT2Z9U0cKoZjSe0QIhAKsEUCjjGLdhbddn/qvCDk+lQ+dM5ajzdWRMfCLNSqty"}]},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]},"0.0.4-4":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.4-4","_npmUser":{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"},"_id":"robotstxt@0.0.4-4","devDependencies":{},"engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.103","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"76a22be8a02a67aca220c858f04776a198342fbd","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.4-4.tgz","integrity":"sha512-cCaMh+bB245FkQkuq9F+PXeSYAqxc8dIft22Yu/04jf8erMqMkfMGoFQPk6Bt6062EW/UemZ1O1otHLKTx+mRw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDqo3KSSHoHzJf94Gpcft4zA2COZQ3D1MSGNhltQzfWMAiB4yF9fO8XfqUym51VjBJv6cP0ILH3vglolKK+7CtLKPg=="}]},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]},"0.0.4-5":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.4-5","_npmUser":{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"},"_id":"robotstxt@0.0.4-5","devDependencies":{},"engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.0.103","_nodeVersion":"v0.5.0-pre","_defaultsLoaded":true,"dist":{"shasum":"2f88adbb0fdc20ef3f33c9ef2db60971335a6989","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.4-5.tgz","integrity":"sha512-EfMnVVUa9t7VQzmHq62Wzgg9lSbV5gdG21ZCbRSdQE5KmEm0W2v2WePOomg9muGy2JKWg1ARhG06NJ8M/Gfg+w==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQD0mKOuzaM3wIJ7YBQbVf4xiQQeKst53HVgCrd10Wgx2wIhAOmhNsWgF/xaT+5QRrgpbBJ7o/LCQEFyvTUSnRm6Tpc3"}]},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]},"0.0.4-6":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.4-6","_npmUser":{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"},"_id":"robotstxt@0.0.4-6","devDependencies":{},"optionalDependencies":{},"engines":{"node":"*"},"_engineSupported":true,"_npmVersion":"1.1.16","_nodeVersion":"v0.6.15","_defaultsLoaded":true,"dist":{"shasum":"ede86dc8fad8c0562b4ab97d2fa103fae79fba83","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.4-6.tgz","integrity":"sha512-OJTyZMJ/emxwDIAG/xKGgBqLNVKv8ttcpEZhXNVeEM5uQ8WFnNX+WlR9Tr6LkYte3S6i9yyB+X0nugcitkvrZg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCTuD7gt/wwtFYZx8gorm59ii9MJNR7E/90bnRGYlTF6QIhAOdzq/FerXuPtGCEmHMjFm+6r3dYBUDy7EKGLSWXy8bC"}]},"readme":"robots.txt parser for node.js\n===\n\n  - robotstxt is written in coffee script\n  - robotstxt is currently alpha\n  - robotstxt offers an easy way to obey the allow/disallow rules listed in the sites robots.txt\n\n\nInstall:\n\n    npm install robotstxt\n    \n\nall examples use coffee script syntax\n\nrequire:\n\n    robotsTxt = require 'robotstxt'\n\nparse a robots.txt:\n    \n    #robotsTxt(url, user_agent)\n    google_robots_txt = robotsTxt 'http://www.google.com/robots.txt', 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.iamnnotreallyagooglebot.com/)'\n\nassign event handler after all that parsing is done\n\n    google_robots_txt.on 'ready', (gate_keeper) -> \n      #returns false\n      #note: only the path and the query of the url gets parsed, everything else (i.e. http://, the domain-name) gets ditched\n      console.log gate_keeper.isAllowed 'http://www.google.com/setnewsprefs?sfsdfg'\n      #returns false\n      console.log gate_keeper.isAllowed '/setnewsprefs?sfsdfg'\n      #returns true\n      #note: only the path and the query of the url gets parsed, everything else (i.e. http://, the domain-name) gets ditched\n      console.log gate_keeper.isDisallowed 'http://www.google.com/setnewsprefs?sfsdfg' \n      #returns true\n      console.log gate_keeper.isDisallowed '/setnewsprefs?sfsdfg'\n    \ngate_keeper methods:\n\n    #asks the gate_keeper if it's ok to crawle an url\n    isAllowed url\n    #asks the gate_keeper if it's not ok to crawle an url\n    isDisallowed url\n    #answeres the question, why an url is allowed/disallowed\n    why url\n    #if you want to change the user agent that is used for this question\n    setUserAgent user_agent\n    #if you want to know which robots.txt group is used with which user_agent\n    #per default uses the user agent set with setUserAgent\n    getGroup (user_agent)\n    \nrobotsTxt methods\n  \n    #fetches parses url with user_agent\n    #returns an robots_txt event emitter\n    robotsTxt(url, user_agent)\n    \n    #blank robots_txt object\n    blank_robots_txt = robotsTxt()\n    \n    #crawls and parses a robots.txt \n    #throws an 'crawled' event\n    blank_robots_txt.crawl: (protocol, host, port, path,  user_agent, encoding)\n    \n    #parses a txt string line after line\n    #throws a 'ready' event\n    blank_robots_txt.parse(txt)\n    \n    \nrobotsTxt events\n\n    #thrown after the whole robots.txt is crawled\n    robotsTxt.on 'crawled' (txt) -> ...\n  \n    #thrown after all lines of the robots.txt are parsed\n    robotsTxt.on 'ready' (gate_keeper)\n  \n    #if something did not quite work\n    #note: it's a good idea to always implement this error listener \n    #to prevent strange looking error messages in case there are internet connection issues\n    robotsTxt.on 'error' (error)\n\n    \n\n**NOTES**\n\nthe default user-agent used is\n\n    #robotsTxt(url, user_agent)\n    Mozilla/5.0 (compatible; Open-Source-Coffee-Script-Robots-Txt-Checker/2.1; +http://example.com/bot.html\n    \ni strongly recommend using your own user agent\n\ni.e.:\n\n    myapp_robots_txt = robotsTxt 'http://www.google.com/robots.txt', 'Mozilla/5.0 (compatible; MyAppBot/2.1; +http://www.example.com/)'\n    \n    \nif you want to simulate another crawler (for testing purposes only, of course) see this list for the correct user agent strings \n\n  - [List of User Agent Strings] (http://www.useragentstring.com/pages/useragentstring.php)\n  - [Googlebot] (http://www.google.com/support/webmasters/bin/answer.py?answer=1061943)\n    \n\nToDo\n---\n  - ready event should also pass a sitemaps_in_robots_txt object\n  - sitemaps_in_robots_txt should offer methods to collect the urls listed in the sitemap\n\nResources\n---\n  - [Robots.txt Specifications by Google](http://code.google.com/web/controlcrawlindex/docs/robots_txt.html)\n  - a web interface can be found at [https://github.com/franzenzenhofer/webrobotstxt](https://github.com/franzenzenhofer/webrobotstxt)\n\n ","maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]},"0.0.4-7":{"name":"robotstxt","description":"a robotstxt parser for node.js","url":"https://github.com/franzenzenhofer/robotstxt","keywords":["robots.txt","parser","web","crawl","seo"],"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"},"contributors":[],"licenses":[{"type":"MIT"}],"dependencies":{"underscore":"*","underscore.string":"*"},"main":"./index.js","version":"0.0.4-7","readme":"robots.txt parser for node.js\n===\n\n  - robotstxt is written in coffee script\n  - robotstxt is currently alpha\n  - robotstxt offers an easy way to obey the allow/disallow rules listed in the sites robots.txt\n\n\nInstall:\n\n    npm install robotstxt\n    \n\nall examples use coffee script syntax\n\nrequire:\n\n    robotsTxt = require 'robotstxt'\n\nparse a robots.txt:\n    \n    #robotsTxt(url, user_agent)\n    google_robots_txt = robotsTxt 'http://www.google.com/robots.txt', 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.iamnnotreallyagooglebot.com/)'\n\nassign event handler after all that parsing is done\n\n    google_robots_txt.on 'ready', (gate_keeper) -> \n      #returns false\n      #note: only the path and the query of the url gets parsed, everything else (i.e. http://, the domain-name) gets ditched\n      console.log gate_keeper.isAllowed 'http://www.google.com/setnewsprefs?sfsdfg'\n      #returns false\n      console.log gate_keeper.isAllowed '/setnewsprefs?sfsdfg'\n      #returns true\n      #note: only the path and the query of the url gets parsed, everything else (i.e. http://, the domain-name) gets ditched\n      console.log gate_keeper.isDisallowed 'http://www.google.com/setnewsprefs?sfsdfg' \n      #returns true\n      console.log gate_keeper.isDisallowed '/setnewsprefs?sfsdfg'\n    \ngate_keeper methods:\n\n    #asks the gate_keeper if it's ok to crawle an url\n    isAllowed url\n    #asks the gate_keeper if it's not ok to crawle an url\n    isDisallowed url\n    #answeres the question, why an url is allowed/disallowed\n    why url\n    #if you want to change the user agent that is used for this question\n    setUserAgent user_agent\n    #if you want to know which robots.txt group is used with which user_agent\n    #per default uses the user agent set with setUserAgent\n    getGroup (user_agent)\n    \nrobotsTxt methods\n  \n    #fetches parses url with user_agent\n    #returns an robots_txt event emitter\n    robotsTxt(url, user_agent)\n    \n    #blank robots_txt object\n    blank_robots_txt = robotsTxt()\n    \n    #crawls and parses a robots.txt \n    #throws an 'crawled' event\n    blank_robots_txt.crawl: (protocol, host, port, path,  user_agent, encoding)\n    \n    #parses a txt string line after line\n    #throws a 'ready' event\n    blank_robots_txt.parse(txt)\n    \n    \nrobotsTxt events\n\n    #thrown after the whole robots.txt is crawled\n    robotsTxt.on 'crawled' (txt) -> ...\n  \n    #thrown after all lines of the robots.txt are parsed\n    robotsTxt.on 'ready' (gate_keeper)\n  \n    #if something did not quite work\n    #note: it's a good idea to always implement this error listener \n    #to prevent strange looking error messages in case there are internet connection issues\n    robotsTxt.on 'error' (error)\n\n    \n\n**NOTES**\n\nthe default user-agent used is\n\n    #robotsTxt(url, user_agent)\n    Mozilla/5.0 (compatible; Open-Source-Coffee-Script-Robots-Txt-Checker/2.1; +http://example.com/bot.html\n    \ni strongly recommend using your own user agent\n\ni.e.:\n\n    myapp_robots_txt = robotsTxt 'http://www.google.com/robots.txt', 'Mozilla/5.0 (compatible; MyAppBot/2.1; +http://www.example.com/)'\n    \n    \nif you want to simulate another crawler (for testing purposes only, of course) see this list for the correct user agent strings \n\n  - [List of User Agent Strings] (http://www.useragentstring.com/pages/useragentstring.php)\n  - [Googlebot] (http://www.google.com/support/webmasters/bin/answer.py?answer=1061943)\n    \n\nToDo\n---\n  - ready event should also pass a sitemaps_in_robots_txt object\n  - sitemaps_in_robots_txt should offer methods to collect the urls listed in the sitemap\n\nResources\n---\n  - [Robots.txt Specifications by Google](http://code.google.com/web/controlcrawlindex/docs/robots_txt.html)\n  - a web interface can be found at [https://github.com/franzenzenhofer/webrobotstxt](https://github.com/franzenzenhofer/webrobotstxt)\n\n ","_id":"robotstxt@0.0.4-7","dist":{"shasum":"311a75a369dcb637e579a3d8362e94133753ddb3","tarball":"https://registry.npmjs.org/robotstxt/-/robotstxt-0.0.4-7.tgz","integrity":"sha512-CewBEmrFEYrH2ew9E3sKsGiPGLR21NDY/0RnyOMhACuGNuvWASBqEaePPvWRlhPmIVbvLVMu0JQDAiod7VgI5w==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIEAHTssACbbHYDlyR0F8DBfd/CSevzDkd22dqXwaXoAxAiAleSkPhPV1pNDg+wYzl4cvRAhpoCO6GoR3UFfGnW682Q=="}]},"_npmVersion":"1.1.59","_npmUser":{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}]}},"maintainers":[{"name":"franzenzenhofer","email":"f.enzenhofer@gmail.com"}],"time":{"modified":"2022-06-26T13:46:51.954Z","created":"2011-05-26T10:16:57.110Z","0.0.1-0":"2011-05-26T10:16:57.876Z","0.0.1-4":"2011-06-02T16:22:52.338Z","0.0.1-5":"2011-06-04T20:32:01.798Z","0.0.1-6":"2011-06-06T11:41:38.113Z","0.0.1-7":"2011-06-06T12:03:08.759Z","0.0.1-8":"2011-06-06T12:17:24.972Z","0.0.2-0":"2011-06-19T13:32:22.579Z","0.0.2-1":"2011-06-19T13:57:42.816Z","0.0.3-0":"2011-06-25T13:49:24.974Z","0.0.3-1":"2011-09-17T16:01:32.588Z","0.0.4-0":"2011-09-19T13:35:01.061Z","0.0.4-1":"2011-09-19T13:42:13.048Z","0.0.4-3":"2011-10-23T20:53:50.390Z","0.0.4-4":"2011-10-28T21:05:55.665Z","0.0.4-5":"2011-10-28T21:07:58.208Z","0.0.4-6":"2012-06-30T12:51:41.533Z","0.0.4-7":"2012-10-15T08:52:14.354Z"},"author":{"name":"Franz Enzenhofer","email":"f.enzenhofer@gmail.com"}}