{"_id":"robots-txt-parser","_rev":"21-a3db88a659ec5ed4af7f89648e7626b1","name":"robots-txt-parser","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","dist-tags":{"latest":"2.0.3"},"versions":{"0.1.2":{"name":"robots-txt-parser","version":"0.1.2","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.\r ## Installing\r Via NPM: `npm install robots-txt-parser --save`.\r ## Getting Started","main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/index.js","test-robots":"mocha test/robots/index-tests.js","test-parser":"mocha test/parser/parser-tests.js","test-util":"mocha test/util/util-tests.js","coverage":"nyc mocha test/index.js"},"repository":{"type":"git","url":"git+https://github.com/ChristopherAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"ISC","bugs":{"url":"https://github.com/ChristopherAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChristopherAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","lodash.isfunction":"^3.0.8","simple-get":"^2.4.0"},"devDependencies":{"chai":"^3.5.0","istanbul":"^0.4.5","lodash":"^4.17.2","mocha":"^3.2.0","nyc":"^10.3.0"},"gitHead":"81d234cf0356ccf37ec65e3214b0d97b70bd864e","_id":"robots-txt-parser@0.1.2","_shasum":"a7d99e8c6c9fe35d1084d63274bbcdd89c8e1d75","_from":".","_npmVersion":"3.10.10","_nodeVersion":"6.10.2","_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"dist":{"shasum":"a7d99e8c6c9fe35d1084d63274bbcdd89c8e1d75","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-0.1.2.tgz","integrity":"sha512-EAMy/NtSacrH3VJj8YE7K10NbzudraV28FGvSQ1XxLiofiktk1yo1pl5A444OiYSemD+0l7NlStWhwh+DG+FXA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCM6XJ13JUsoW9AK2xUOa3B2isG6coHYDYd/AJoGHjO6gIgbgfZ/K0mGo+lNA6qcr1fIBX7XgsIyhJWFC1VTivrPcA="}]},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"packages-18-east.internal.npmjs.com","tmp":"tmp/robots-txt-parser-0.1.2.tgz_1493586870209_0.00831022672355175"}},"0.1.3":{"name":"robots-txt-parser","version":"0.1.3","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/index.js","test-robots":"mocha test/robots/index-tests.js","test-parser":"mocha test/parser/parser-tests.js","test-util":"mocha test/util/util-tests.js","coverage":"nyc mocha test/index.js"},"repository":{"type":"git","url":"git+https://github.com/ChristopherAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"ISC","bugs":{"url":"https://github.com/ChristopherAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChristopherAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","lodash.isfunction":"^3.0.8","simple-get":"^2.4.0"},"devDependencies":{"chai":"^3.5.0","istanbul":"^0.4.5","lodash":"^4.17.2","mocha":"^3.2.0","nyc":"^10.3.0"},"gitHead":"81d234cf0356ccf37ec65e3214b0d97b70bd864e","_id":"robots-txt-parser@0.1.3","_shasum":"a477c2486daff11f006c324871cef2721c8ded94","_from":".","_npmVersion":"3.10.10","_nodeVersion":"6.10.2","_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"dist":{"shasum":"a477c2486daff11f006c324871cef2721c8ded94","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-0.1.3.tgz","integrity":"sha512-2+ZBl+Qv8D7pQaTFZtOV7NskudbjTiOn3zq+OmYYlPOisw/QznAjMZZNcQtPNFxrKsFl1nOuckFXtl82X2saxw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIBzagHR6NBo4tOIq8n8cgTRt1jA4FSr4x8azrJeofh1cAiBCGFkGgRnL30v1Uh6unJgOIxuQ/IJ2fkTsU0dFqAtjYw=="}]},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/robots-txt-parser-0.1.3.tgz_1493586951731_0.17496801377274096"}},"1.0.0":{"name":"robots-txt-parser","version":"1.0.0","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/index.js","test-robots":"mocha test/robots/index-tests.js","test-parser":"mocha test/parser/parser-tests.js","test-util":"mocha test/util/util-tests.js","coverage":"nyc mocha test/index.js"},"repository":{"type":"git","url":"git+https://github.com/ChristopherAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChristopherAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChristopherAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","lodash.isfunction":"^3.0.8","simple-get":"^2.4.0"},"devDependencies":{"chai":"^3.5.0","istanbul":"^0.4.5","lodash":"^4.17.2","mocha":"^3.2.0","nyc":"^10.3.0"},"gitHead":"f68f986ab782f2b64d1a76510d205a253eb7d5e9","_id":"robots-txt-parser@1.0.0","_shasum":"3690024c150dea497cc87e62e930d2782677087b","_from":".","_npmVersion":"3.10.10","_nodeVersion":"6.10.2","_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"dist":{"shasum":"3690024c150dea497cc87e62e930d2782677087b","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.0.tgz","integrity":"sha512-ZaL0qZO4xt5F7gPp/I+mTaoMadln6kP7VH/6zu3FUrD/2jdW1uKmfh2u4nNS/osq7yzpso2VWu2AA+gObREvXQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDWE/+1y8BD1g04K3tcyhjy+cjamWfTviQr5OGOSmGMzAIhANzPSEn+fRXgaRIH5yFenUUdMH93weDsK9GGowRPKi7y"}]},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/robots-txt-parser-1.0.0.tgz_1493587945799_0.6113960410002619"}},"1.0.1":{"name":"robots-txt-parser","version":"1.0.1","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/index.js","test-robots":"mocha test/robots/index-tests.js","test-parser":"mocha test/parser/parser-tests.js","test-util":"mocha test/util/util-tests.js","coverage":"nyc mocha test/index.js"},"repository":{"type":"git","url":"git+https://github.com/ChristopherAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChristopherAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChristopherAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","lodash.isfunction":"^3.0.8","simple-get":"^2.4.0"},"devDependencies":{"chai":"^3.5.0","istanbul":"^0.4.5","lodash":"^4.17.2","mocha":"^3.2.0","nyc":"^10.3.0"},"gitHead":"4ef638cdc7520d93cd1048d147760cc9d67a0bc0","_id":"robots-txt-parser@1.0.1","_shasum":"402f1e3b368418de70d84cdad2895d5e1391549d","_from":".","_npmVersion":"3.10.10","_nodeVersion":"6.10.2","_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"dist":{"shasum":"402f1e3b368418de70d84cdad2895d5e1391549d","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.1.tgz","integrity":"sha512-7CxKFjqBeGSg/11UAVBlcX+lSAkGsHEpvMl4gpu6iL9ketYD/FVbFkhNzOpmM9yyeIuD7fK8PPD4DHyXoQykKg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIE/GuD1LYszfPrX/q09UdvzkUFrMNdFT4yLm9ulKb92KAiBdVdKuZT4oSSEio1kWfvGlIxZ47XzDn67BvosjDwFoKA=="}]},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/robots-txt-parser-1.0.1.tgz_1493669596102_0.8145050073508173"}},"1.0.2":{"name":"robots-txt-parser","version":"1.0.2","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/index.js","test-robots":"mocha test/robots/index-tests.js","test-parser":"mocha test/parser/parser-tests.js","test-util":"mocha test/util/util-tests.js","coverage":"nyc mocha test/index.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","lodash.isfunction":"^3.0.8","simple-get":"^2.4.0"},"devDependencies":{"chai":"^3.5.0","istanbul":"^0.4.5","lodash":"^4.17.2","mocha":"^3.2.0","nyc":"^10.3.0"},"gitHead":"02ec1fd04888934ecc79104adaf8f071ecbad596","_id":"robots-txt-parser@1.0.2","_nodeVersion":"14.1.0","_npmVersion":"6.14.8","dist":{"integrity":"sha512-3UMXkyygZBuZBJs14emjoA6xpt8jrgB+Bxkr9+yMZSVRH6JLCcu2nojNNo5VJcwZDmAyX7BM8EfN0oy64Wr+fA==","shasum":"c2f56a4268d163aca1d760393a55d0f04e054085","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.2.tgz","fileCount":39,"unpackedSize":46012,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgYypQCRA9TVsSAnZWagAAsaQQAI6hIm/dlEnvMzh+B5Xz\nZJc2e1l2PO/yFNMRWKegEtGeqKpTTWHivRqE9PMxNy2f2tom6OUpdMMQTeP6\nYqHO418ofUQwOPXpd2kjxWwBQ80BIGdKj0cqjMUaDj959wsuhk5NrCkugLlH\nhO0FIi9gp3XO+Z+qlCE0UMNJDdnyn6pyqHDxP69KMcPj/JV0NuexpkOhZqAx\nqfz4iUbzI7zhEGdacw8ujNCkcEEoYKBOmmwp+j3qHHj+FTT41Z8flpw25YiV\n+NOf7CeUh5U5dNaoEZIdQMW1gUNPtYhU0heSK0irRi4P12A/nv5M+17sSjSF\nXE4/4gXuA0kgigEz3nwUn+4xHKQaTrLdEdPlmtyTNzbPnx62uIFV39hfuh+1\nazxYnWNtrvPWy+ul5uvRVKYZkggew5KIyCGVVHaOd/Xj9cAUbr065EWBI2q2\nedWEmvXHsA4RCDkauwL0afun9iNyHAmwFsSeB9QycxzR2arL04Ji5mA71FYn\nmSsS9HIVnhRgpXP35qH0NADKdccj6MeOIh/pX6g+xqo3MDgdjNqQG+bStX7e\nZl3b/xh4VqdqnY9PCfUEC370qpeQWzZCvsLgt1v6xfhIzgBtQwxtZzNwxNei\nS4pnkR7qjswzuIXsID/arNFQCjmH9cJvxQTgc/W6j/wRzeFZrsNd+ByqNjMF\nkScI\r\n=kb5Q\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCID2FV7obafuhuI5/vjdrBcEPzLkDQr/4gVPKp5KxPdR8AiEA4l67tlMXDrl6/r/9khM0P1MA0uwFpTxBc7i/PWGx+P8="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.2_1617111632132_0.5219556670539331"},"_hasShrinkwrap":false},"1.0.3":{"name":"robots-txt-parser","version":"1.0.3","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/index.js","test-robots":"mocha test/robots/index-tests.js","test-parser":"mocha test/parser/parser-tests.js","test-util":"mocha test/util/util-tests.js","coverage":"nyc mocha test/index.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","lodash.isfunction":"^3.0.8","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"f834932b97b667315ff28a9cf61a2f6c30d7e6d9","_id":"robots-txt-parser@1.0.3","_nodeVersion":"14.1.0","_npmVersion":"6.14.8","dist":{"integrity":"sha512-PVSO8s11i+lpBatTP8WcjTY99tBygas+45VABBOfgQaAniOyqKJslw6/nABde/HfeAyOtxKJrB8zU0BPX8JaIw==","shasum":"d72e918b3d659272982cccc8be8d70de22607ffc","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.3.tgz","fileCount":8,"unpackedSize":22923,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgcbvjCRA9TVsSAnZWagAARfsP/1c7T8JPhlxRetENHUm2\nosmL96oi6nPjl6fPFLwLslL6Sgf6RgEfB+d9xfStrTqzNEMgRUENZ5KEUKQY\np1I9n+xDOY3L69P45D7UCMpeUqnKmCx2DwnjuRaJORsTbTuZXTwjhMdR8sUx\ntXqo+BNzuBJEEvK2AoPShyMPm3bDIiDbLXozzSXPLxPMniX7+9YmNvWw2nVx\ncV8Hv5hdKXiB7pvW185nPbc0UCN7MdL3/VVmhIFS67MbBmmxOk7/EWyOkcvW\nUBpu04Qw67vbgmp0x/ZPPf9urS042sTYZcva+xPdymSpkK699Jsd55U4IwAy\nUgSmMyqMgqrQgUxQu0FMePWv6ZtCzdAlwIB8AYdvKEdVXJxBJbjhlWN+QO0a\nSWe3zbB2sz2bYwDspQ61Pd9jElDZOjIcA7DrA9QXgRT2JitTXdXjviVTGPPc\n+5yN4XfsBF6n2XbI3Bz+aENKGc9Zz1TDOaOL53ZcpM/PmQGG4Yqm/3Gl2RpO\nNIg1wtJTkMnLsPvqiV/T+mTpZZgAb8rNUwIExE5OZVc9BKzKXzLdPs7EuGzX\nFl6Q1hk3tZlLTSw884ZOiPj/Fad8ZnXWooaRsdpQfHl/Wk7QuiHj/kHY2Eo5\nkkjgvUjQv7MIBpXeW3wHuk6H7ZfqV8xquE9P7z5MVDceOacuqWtD8RdvLo6W\n0e1c\r\n=eTo2\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIE32+eCJdoEHiKRXxzeYyKKk/Jh1Ufl0CRgmq1RJw2N+AiAvgWBpcS4JP6kMEAa3Rpyj4F7WLAhSrBMA9Gd15hqz4w=="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.3_1618066403332_0.37726376698409547"},"_hasShrinkwrap":false},"1.0.4":{"name":"robots-txt-parser","version":"1.0.4","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"c549401d6f035b0e05b3045cca0ad5f2f51811ff","_id":"robots-txt-parser@1.0.4","_nodeVersion":"14.1.0","_npmVersion":"6.14.8","dist":{"integrity":"sha512-AuKM7FIFY7V839guvSEaJNXY9rob7Qhv3ZsESj5BvhlvoKmoCvtrG1EUaeGmifQwFXhto1qDGUY5TRUmhT8N8g==","shasum":"bb034150a181319dd65a9bbb445af6ffb72e0f3d","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.4.tgz","fileCount":8,"unpackedSize":23410,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgnPxvCRA9TVsSAnZWagAAydcP/iesM2NxRPC2KIRrKfZp\nbd+1mJirUG1StuTf3Dx9gnkDpbMBWYZFm6FsXycxmxe3vW1BxzTG/b0cm52x\noUtKESWQ4IXW9+wcOgyBF6YEJOhoXbgPVx+UdX+vXo6bHO3XmQx6Nchulz21\nxLiuu/9aJJxOPX/t632HvWNWLvReEQnJR8SEkrK3hX2wee8eJE2IaHxdW1qk\nHzvunHVS7kcjuAiaOTq7m1N7IGuNlMX8TNcfDFGLfS7tadsOPZAu4pcF0s/G\nZSJnUxKG+SlpxQwOUGIBLZRwXyFFV1MvCXwCegpic/m9lUqZZsu5R/kY3otW\nJYMtIJf/ZiYslQQGbcoKycDz64xcb2j8VYpgwgB36vx3khEZJd8RrctATTXz\nguMkITor6NzCiNzVuAmFUKW+MeMtwwx8Y5dStSe1LhgCiSvKho97/sEVREL+\ncyS7khfAdFR1UmFxzw7C+GBlBh8WZD3aUix0VPv+SVpAUxeSnVfiLQ1xGY16\ncKa9Cw8pA7gB4KVcwsaWpOBHk5AfU5Mbp6iRjGOkCfBoCls8F5iRStkoWlc5\nWJ7ahZ6APZqNO2vfJdwsCIBmMIwojyqE9qWDa3uChfOD6MTLzhTsoQz3Mbgh\n2JioBfKMuFTazW2b2MCB1tqqe+9t0G8qD9O7oLAhIaa4X9nOSwsZH+lxvgD+\nqHPZ\r\n=hZDR\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCcjajdVYIto6D7Z/JoA2SH066ssaqEE9t3Cv8BrNdHEQIgQEUfKZYqgGcb4tKc0VKV3kFn+VGv/qQqUqAj1BweIGg="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.4_1620900974746_0.03460116418221393"},"_hasShrinkwrap":false},"1.0.5":{"name":"robots-txt-parser","version":"1.0.5","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"36b4d9baa5d1c45b91e945ff6c74f81e9e6e4851","_id":"robots-txt-parser@1.0.5","_nodeVersion":"14.1.0","_npmVersion":"6.14.8","dist":{"integrity":"sha512-fKRzd2ovnK5utn2dzVFxbpmn8PY0s7szdElQmyyfRYPADRzvLbVZPe3nE3VIT6h9BFlNNj/lZHZccJxdliHTVQ==","shasum":"3404c40c608df26f80251dfeae0709eb37b0addc","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.5.tgz","fileCount":9,"unpackedSize":26773,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgoTypCRA9TVsSAnZWagAA3rsQAKHEnkt/J5v+7XmPdIQG\n30NUMKu/0GxjTa1BmdfnpIGP4Z9R45fNq9TMJVBaq96DjkngQcUZ6BK/PrNB\ngEFtCL8Ffc88TqYv6P6VBQvJa6Gn/lEwDNWXBX1F74i0901NeYuvsEvxmH7A\nh1JAb1aUVr7RQimf1kGQQd2DvVN7Nr04Ea5Sfq+dNePcPvIrVfQaojVmahEg\nK7AdZwX31ReqEfrEPbjPed3MOZAZlyEH+DDu0jiZ9hgAcNSZ5mBfdvXLBwtD\nx2M07DtIcAhJbn+0r0FTGlXm1GY1PDkFKJi+9DxOLQseTWqtjgjE/ojTvO0g\n+/VcReZlg9Ak7TJWXDCSfPwIKT9WrUBUAev21Dxkz+CM1d+6mSsbKRfoUy0I\naPMb7CpwyvfkzOIWIKzAwDkdJ1Cp4kSevvAhb2hrD5/Uh71bkFP2Hr98upWo\nRVWAohTCaycxh5RJJqqA/GtfikN7yUpRLxqIhNSkpLiEYL2ruymbrO8dVp3+\ni8x5CvjKQpNaFXROQEBJPndNK2e1QJ1olmOeEjT5g+eeVAukQaIdHV+krbWN\nnWN01okJNcGxw/dPytsB2APWXRHLVe8GnydU1lMYTzHmODFqsPLfgTZGmHYj\nha2w0re7ZSei8bdk3NIsJz3hOjtH/0eo4VsQsI5SWCaC0Okx+J5YccdACfPR\nlzL7\r\n=4y02\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDSD4Su1EQuczcaMEZqbgieY/FQsDHpAq2Xkdw2fJTpHQIgOu0iAtAm8GBCSztJbuv0Q6bVlHGbA5mToJHjfgMipfY="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.5_1621179560600_0.5269635431088955"},"_hasShrinkwrap":false},"1.0.6":{"name":"robots-txt-parser","version":"1.0.6","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"9cf072c645a5ea2d9275121a7d39b215851a5fc7","_id":"robots-txt-parser@1.0.6","_nodeVersion":"14.1.0","_npmVersion":"6.14.8","dist":{"integrity":"sha512-jTYet4Ke53m2QuHFyojauBOeFj0D/nMwl/jQGE3f9ftenwSCCnVSDZlGwfxEco5VPkqUoQLP5HynOB4Dgg7mNg==","shasum":"7aee34bb6b902667913b0b3401f02bd1d5e4014f","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.6.tgz","fileCount":9,"unpackedSize":27199,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgoWVFCRA9TVsSAnZWagAABFkQAI1z7/SdhG5py/A9vzf9\nUQM/eWzycmhLzTxbFDmf0UjomDM0gb8pHYleL2Vrtyai2DHachxZQ05OUjRM\n4YfBHuoXYpaGtBc8YjZT4hTktdMyyD33tefZ9iyxrTwbvXPcpzVqNnPB9QU7\nxHxDgP8pkil1ojaPHn2Qf87pmY2Hks+bP2Nmwb0Uq2BchM08yYLGGoQhHuOE\nusLODQPizaeOYhMFabRvHDG6cI9a8d+gsFBqEme0c18+amjV8ogwQ2oCTxgk\n7lNEySayvEtNCI0hppYaPSz4LJGPEsLK66R+JHFspfEjzgKBXYRndEhScaLa\n9FgCgU6XpOUV62WdxPE0oXQMa9Bv6ritt1IgwCrAV63HPCbgOvcKtHsmnADG\ncdsakzSh3xJck2xSH+j0rvcM4Ksry0WmkWT8WbxR7DW0YKtYqLo8GMtDbb2R\nBcdznkdUMykmDjCQ8gxRFBaHEM2DSKcbPCn1S2O6UP1nSwhj6okr8ebXLJV/\nL/TR8HXT1c42yUbb/pQtheTkoMLFrQvH8nCpbhClx9ZSzia5cQeYJf9q7QEA\nk4jmJNbm+8DrCrutNWY1Oub02cNp5OSdaDsvikzKbU99XmHkc0OxscvTVYa+\n6gl0czKUNnjIv/2Y4EI1i0sze9VJ7Z0jw0B1L39inLPofdkZzyt5mi9Pu1/p\nPsOv\r\n=yT7k\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIANH10sNMon7AVWi2+O61yPsb2QAIMDaLAmcLs+6PDUQAiActRhriwA9wsDlUle+JKaaz6/FJoJOSL5vf00G5i4p+Q=="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.6_1621189956610_0.5134022527334283"},"_hasShrinkwrap":false},"1.0.7":{"name":"robots-txt-parser","version":"1.0.7","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"64711628998b853ff8ff9a8c4481ec3259156956","_id":"robots-txt-parser@1.0.7","_nodeVersion":"14.1.0","_npmVersion":"6.14.8","dist":{"integrity":"sha512-AVbEmNFIrN4ZxkajSXRu7+2F5fZo9VX+uyGg0MjAE/XNB5GU1gtb9GCmeERtUj2pRRsfzN44OmCuC1ePQ7DIQA==","shasum":"8eff5ceac61fba933702d26ba834f2755ca62029","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.7.tgz","fileCount":9,"unpackedSize":27199,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgoWXvCRA9TVsSAnZWagAAORQP/3YXpJwbL7TMlIe8uzrK\nsWqJiQkjQ/yS0FuOoNgcSka/sJIHzdrvonFz4FFRxyBUNY5jtutrGp0sLSJb\nM41NLybAo3Su7pudEwyMU3ubUroKDys4BCDcabriFn4uFEY/pAqMydunfLfY\n740HyivIT351UmO411ftj+n/aSwFsrZF1AaqBq0y18DnutswCMz2JIiBk/W2\nlMd2VFDWiCxybr1PHyKiOB4CHny/XGqipMszPOKmWHyJ527FGxMwS/gbjs8V\ngG+n1Q0LwNTlqVPqMf8RiUbInnh8Rk/0PHVlX3xPehCBKA2V3ZUUnOwZ0VSZ\nKSatdHGMFQLvz1V7VPymJqnJ9DLRv2tx1y0kxqVxPeSDnDsQwOz1jTtwgYYp\nRi1nDQDPGgVRJ0bhqpeAsrnO6pULhga16noo7MSycLzcHYXMKo6b7H7co1v7\nIqWElCwKZ4bxPY3Lx+oGaGvSvaP+vEuNmwcN+TOxI7YIpbpapJkPMYLdgfWN\nyHug0F8T5y8Rti1MZouHz5To7SeeizAePeqi049l8Nn2EnC8HWKINWGonkY0\noTYSiPwLlbR/pLbyKpI3fICnbOts2VbQ9LGFBGTEZLLhkkK2GBv+fOmKl3Gd\n2X6xIf4C+9DVhdEElPCq9/vnWuysA6dssikk83PDsqBE8m3P27lGG93Cfluf\nOmaD\r\n=nOZJ\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCy2fvaUHCTZ5WyvVFfreW1WTTBlMmDnBMFnQerPZ13TwIhAI2/9tkCPuopistaT+VdC7wMIzW0EXNEb+IUXSyWkSGg"}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.7_1621190126953_0.31997282985984254"},"_hasShrinkwrap":false},"1.0.8":{"name":"robots-txt-parser","version":"1.0.8","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","eslint-config-airbnb":"^18.2.1","eslint-plugin-import":"^2.23.2","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"98c60d78929970f6ed30fa76f14ab56a16c0e302","_id":"robots-txt-parser@1.0.8","_nodeVersion":"16.2.0","_npmVersion":"7.14.0","dist":{"integrity":"sha512-7PXlDlIdIdBOW+Z6vKeYbGyBzdFXHUyWPJmDChBF+z8wh3HN4lj+aEUNl4kZ+d1pv5XrbLI1SqVEwU0bheX84Q==","shasum":"3b505a098cd96587c0770a785eeab606db4fcf5a","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.8.tgz","fileCount":8,"unpackedSize":25359,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgp9+YCRA9TVsSAnZWagAAE30QAKKkCAohKsIIQrHTSgdA\nQgFgOByCUlhmIeMb48vd6EjMIXOSEfNJMKQk8WRUsyrLMAJyVVMCK4vZTGty\nltQXmHGNnWhpMPq6e7OZYlTeVFk6bV6a8jRu0XcOSh4OMyZLMJ0RhzMHLmwa\nl+eZikNH4EeeBZgfeH+E7HvvEu/iPu1ju+e+QIrIyY6oaXbYXfpKN1Y9kSTi\nVc1ABds/wirqLGDFysMg9ydaVR9sdd71wy9CCvf4oqqnjNDcfrgQtVxGOx/Z\n2WknNkI/LqQYrZ60BUMMnHlWsNN6ABlephZ8iBY0kgRKLur3hvVr7gAVjH9S\nCaO/lCSFZp3PSC0hG6XVkEebh3Cs5Pne5Mb21Nd1+YWkHn28wDFiw3GzXkk3\nQhkiz5yL2qD6DJdyGlC5A3AFDFiH1cjXegXL6CyW1CWaZvjZOSgUZVix4uMW\n2sd8TMpvHmHrC4WGTcEZrCTcKwZornmJPoOUIOQdhmiR0doi4AE3Nt+Lt79a\nf80W8BslUpSESPS2lvuEqHm9mZynqyEW+dDUwSXDhEgTvZf5rMH5pH7O+VpD\nrenrIJQ8n3hxbsAiP1ovtR1jbqbdntBPsHcUmNCCkdv0k5xIVbt+skZdIAOT\n4MgxNvq7ccRFfJv5twUgXes8Jcbg2lfXaUEhx1CBi8uCz94UaVg5BMA99gj9\ncdZU\r\n=G/Aw\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDgZhP6mAz0PWeh6S28lGX5/XIW13mGrjLAJKoLXSFqaAiEAjMeQ+okQMQmqaeGifcay4dM5WjhhHYmtPSjRMXthFm8="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.8_1621614487631_0.7187255759645372"},"_hasShrinkwrap":false},"1.0.9":{"name":"robots-txt-parser","version":"1.0.9","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc --reporter=lcov mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","eslint-config-airbnb":"^18.2.1","eslint-plugin-import":"^2.23.2","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"330614b1fe45915dbe553f3648bc9018094962a8","_id":"robots-txt-parser@1.0.9","_nodeVersion":"16.2.0","_npmVersion":"7.14.0","dist":{"integrity":"sha512-QQ7UT8YfZXnd1kdoBgs+C05EXmHy6lhbDXAYiyM8A8dD1SPu+xUFw1DFHpBbqLWzsgbq0wmzDBgsreaZJOlk1w==","shasum":"750742cf1282b2d27fd88083cae2544c7bbe936c","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.9.tgz","fileCount":8,"unpackedSize":25673,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgqkb4CRA9TVsSAnZWagAADqcP/1JwXCAsqDJRf6Rp/AF5\nPRUOQgMhPl6XCgi0M0wjBPtmgtQ3SY6JR+sXr//mY0Nsh3nGeYUrRCIPDImu\nl3LMSbX36YDu4u7e+NCROJKR+qfCZxv3XHoplP4Kb/BzkB5Rh5VBsxye58MS\ngABHG8w4YvCmlrPMAsnK4Kp75NdvwJ3zgiPBMa7u/C817bvWp1HT8EzvL6VT\ngdgQ4wLZ7BW3G1W42uexmPjO4briJ7TBUzyu5qUxUl/GkHgHXo+hAEExKiV3\nMuZ6peTeh4NlKul2E0kIfh3lsCPexnAZNlMf3fX6IYQn1/OMDfw7jInXOSUd\nTucivfUTJI5CzVW0zy2QZ61PXH0HTPkWeRdcU0WJGvRtYCYSnY+EJn1ZSB+n\nBhhsO0YOqW5nKBep50+j3CuLPE5SR1cLeqRka4cfpyohLCBdTgIZ3ArzghHH\nrVlx41URJ5/aMYkD4bydqPUxq3s3DJrIMaNrceghkpyzI8RBcq6uus1QkFiX\nGmlV9nNe96Rdb5mXXIak8OT7mgZYaUz6Y6zxtsiBwWCx7m5QJ9znWuH4XeiN\nYfMt5WGCknsijOG8yRZO+K/+Pv2HUkp5tHlfRIUsw12wqG4BHIKr8ZOZVyJs\nDEroHfdIzxXaoZT8X3twdbNHVl7p/bCkP9/+ORegnJLKALqsYGrFLwFymFbj\nSrjs\r\n=iunY\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDlZfmfSmeTm3HztwE9yg8cIAeo5i/mHmkiKidzodSD3AiBZ+NBlTKlk6QfZn0c5zall43+NBlvpTyQWYwxqRzBy3g=="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.9_1621772023691_0.4900094454042436"},"_hasShrinkwrap":false},"1.0.10":{"name":"robots-txt-parser","version":"1.0.10","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc --reporter=lcov mocha test/**/*.js","local-coverage":"nyc mocha  mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser#readme","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","eslint-config-airbnb":"^18.2.1","eslint-plugin-import":"^2.23.2","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"6e41fecf907cd60440b64297c0665a2c4868f40d","_id":"robots-txt-parser@1.0.10","_nodeVersion":"16.2.0","_npmVersion":"7.14.0","dist":{"integrity":"sha512-HtgY7AwZYJpZXWFP15hhKVY2wCqs690qkJN26n9VVPfxnP0Qnmpgf5P25k4FtDUeaND/QQ+DKjHZ1OnfbxWCnw==","shasum":"a4f11852ce53547e12e7b42fb5e29ad4dbcc3a98","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-1.0.10.tgz","fileCount":8,"unpackedSize":25730,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgt994CRA9TVsSAnZWagAA7R8P/33Q7lkyzOOaBmzrCj3J\n25Brg0vVNYUT9vUd8/PRzkOPeWtKeXBVL4qFbq39wrJFbJeBkmnxJ3bENBWP\n0fw0YTU/cTEmKn+tSOBDh9qA1JxozCuriPO6ZAzan9NEoDXcpk+cOa8TSq0A\nz8CKnPx9rrtypM8RY/F41tdJFV1Fp207osFc77gGc81A11qPuFEmFRjJ1PbZ\nAaBs+Hcj7SMVMpl/X+05m3qUJvuhRj4y3CarolPhDockKJONHUA+15hhd3u9\nDEDaiVMGwon3TU3FcK62yvOWgJZUpGHRuKldw10dPWQD5Vz7TDllkB07SEa5\nz5k+gskTCagqiu7AG51azs5Q+ee3oLevxAjkxPGqFsaB4rQgSiyll9XDhXXk\nOJvyexz4bmOd/rMOFbsX16LKgW8xfxnJxMHs5naqRVPE6QXc8UxnELFae2hS\njrWnCP7fkV1SFRY4+TPWyGge6y9n8iepf7/A6MTFKRhia5LdM7T3ZIHYKWJO\nZQlTVhmQKnNAxA0qDrqUFqICyxvPsd/CcHBpDeqD011AvRBJXt2KMLcmZLRk\nS7sgkUo/3gzIAOWtvuX0K2am30oBXxAjYIBH3W580pKzl+d0tkumBPEUjbWE\nzHfnpC62IzWxAUohVQ4npdH+JFUS3Nf1NmqUiv9232TyHjkBbWSOHYEDBLTW\norb5\r\n=wTQR\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIFbrM7ayzRfZI3tmwsIQNMgeyRcyUTsLz+Us8xGP7a0DAiEAg5kZNZYOau1rg+djFmtkobFMs2Hh3UjxE527PrMSha0="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_1.0.10_1622663031973_0.6008187708501147"},"_hasShrinkwrap":false},"2.0.0":{"name":"robots-txt-parser","version":"2.0.0","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc --reporter=lcov mocha test/**/*.js --timeout 10s","local-coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","eslint-config-airbnb":"^18.2.1","eslint-plugin-import":"^2.23.2","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"e6d62c77347a627ea5ff71f1ebbb09170249d9b2","_id":"robots-txt-parser@2.0.0","_nodeVersion":"16.2.0","_npmVersion":"7.14.0","dist":{"integrity":"sha512-jR5rxibRUQ5vbUAJuf21E81rMmxzBBV8yLezjUpGBBDFYk3St7jnJALjNN1EkL5unAuZa1erigXV/bUv4jDucQ==","shasum":"9db53bf8d6650df9a94f767fe84523653a3ff4bd","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-2.0.0.tgz","fileCount":8,"unpackedSize":25332,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg0PQhCRA9TVsSAnZWagAACoAP/AxaCzkBIRd4ztxHXolv\nC50Q0sELN40wPOjItCELD5BG4sZO+JkaOjbMcAldWDOqauvVa4U54rRBJVaN\nEzsqhBiz9EBkudYSRnyDoRGbhuX6hMlPjktmQ+W1OA10xcAZT5bzgruGKUP6\nQYIK83NLUpDlRhVz/e01DI0t/KpPo0+Y8eZiPPAh43GKqp70HALjTzOtYPPf\nsTpM2jr5oUrupvIOCfjImc1ZHWiYByXVAYsrD1d1I1arDyr04IvPQyBRFRX8\njTHQkrItnxldgjQ4UL7jXw5eNlUvqaGbUj5C6T2Nvv+u2+o27QeGbvKFofem\n6OHvP1cj6fMHQGNQGFxXng7HPphQpV4NPFNLTDuYE0SBF9K8faQhWakm7BS8\nf4BFyxCE1cXWF99Bs/YB1BMa0WpYoQTi7xdxtQvYNoA8RubBVo5sqfqHx2xQ\nnZUk9zU44huvfy7KdCxOaLEpuZCOGIbh3C1PpylBt6ICUoRBOx3dj+AUMAzf\nu8Yx0XlvtvWD6l7ue4vBnKsruPiq0yxh7Mm1PanGAm6rdiwPSpHccENsW/NJ\nh1b3khihQ0vqsHHmgJQrZRTomE4bL8xIuVg/EB/U1pXSjv82UgJSM+E9R0J7\nxjO1+I30Uvf9unsXKxe2RHXa1cI1KiDGkMzU8UEcKSQj83X2AuPZ+RtDBavV\n+9lW\r\n=i7Xk\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIA5+8dHH/VPknErZI1Rx3dqPpr3X6UPsyiLjT9252mOrAiEAySfFgKiA7LP4I6gNFS5xLX46DjqIwcO2U6p+0nopWCU="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_2.0.0_1624306721301_0.4820194916839511"},"_hasShrinkwrap":false},"2.0.1":{"name":"robots-txt-parser","version":"2.0.1","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc --reporter=lcov mocha test/**/*.js --timeout 10s","local-coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","eslint-config-airbnb":"^18.2.1","eslint-plugin-import":"^2.23.2","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"90a1b500782fdc7657ee4ca2777feb7e9f971cee","_id":"robots-txt-parser@2.0.1","_nodeVersion":"16.2.0","_npmVersion":"7.14.0","dist":{"integrity":"sha512-yvz4SgD3Fkz99Qd8/m1nYwcM3jJi/bnPtc11zRAuWkXdYciMcnync6Q5NtJan72gKKZRayB6inBe9idEaptMog==","shasum":"6cdde9b2013774de678b96f4fa45c92dec57c4c5","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-2.0.1.tgz","fileCount":11,"unpackedSize":28100,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg0fNjCRA9TVsSAnZWagAASHsP/3anluc/hlYwipQGUsUY\nCcSYsl+RYPmnFf6eovrDVPfMK4+HciFFZDqFr5ESRkjKLCQ0gaG3L09UEcBG\nhNgz8TUzLRLdpaENT4p4R2cQjzs8VL1fy/YBrAfc3KZEpDRZlHa6mT+LXdmu\nOJaZf4FRua8jd5dqDEgwoIbEWI1XaRzoyejFyQL0Zsm2Ho+5jrKRJSBl9dZS\n/BRoJuoaKtUYjug5lFlNT9B+CICzK3H08k+23KS6E/Zv9YlgolgkWgYnYLml\nskKgn9urz9M78jW0q/bUDN0/Lwf6/HBWt62yhpC6MhRF8VhiwLLSlPip+xyg\nq4PZfnc8gobSK9tVM0X2KfYwkAf6qaxonp0qAsLuTJLbynUeJBk1fHP3BZje\nREn2NWrtlFbW3o42tDXriNxS5u5tYrgu5Vc8DqrkoyiKvX50eXTu14Ai5Z6+\nb4sMlbC8SxxQGFKb3GMRWBM80uJiFppip4M0WMUwatHQjBh40j0FyPORVXjm\nBl8Mb+fuM8aJa5efoZ4GqscsGobakx99Y0ml2U9RnNNmj23jyVAEfWDWZqPJ\nbXX43nU6eiqvhm+Ey8OM+LeG0bpCah5biKGOdzgMwrFjk4ItzEGU97NHegII\nRPvTw66SUhDkRxT/EleHCsDHDAlVPNBLQqPHSqgbsvuIQIDWtGvFEEup8d0f\n/3Zm\r\n=CIYt\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIAbBPWgNmwOsBM5AR2YJDcRHBjvylNFIoK/l3C2sDrDsAiEAscUOjawcJSRSxX4k4MbqDz3fkDMgGNUkjuQlEKZX76I="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_2.0.1_1624372066636_0.33996782225153876"},"_hasShrinkwrap":false},"2.0.2":{"name":"robots-txt-parser","version":"2.0.2","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc --reporter=lcov mocha test/**/*.js --timeout 10s","local-coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^2.4.0"},"devDependencies":{"chai":"^4.3.4","eslint-config-airbnb":"^18.2.1","eslint-plugin-import":"^2.23.2","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"a510f1a265b6dfa5901d4400fdc77ae8dde152d6","_id":"robots-txt-parser@2.0.2","_nodeVersion":"16.2.0","_npmVersion":"7.14.0","dist":{"integrity":"sha512-U9ld5MlueqEIEqDzLHE0/evyw9JQQY/OB/L6DPdnljSpZ4Mxec/QlU1tkso0f53IvVR0RH0TGvzmBqdo1cFSvQ==","shasum":"bc39a394a7d695d3ba0504b89928d5ff045c1073","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-2.0.2.tgz","fileCount":41,"unpackedSize":59628,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg0fPJCRA9TVsSAnZWagAAcZIP/ReR+CRLGxgB1PC8fdvN\nU0TmoWUEYkqYDv29Ag1Cpr02s1OyZY+othQmJYTWhpd3usnf0xELK1C4kQyh\ne/PkgfUA4zE+X2vLrRGiIvFB4He1MsfVSOIrvbZbl+r22GFrqBXIRFA1veA7\nOm7y/vabkR5eo6TVZXPDdWZ2XUu0jjnJLbqB16/UE62U9QWLMjavHHwcydqc\nU76fj4Fjj3BCRfHDOQuEFct7nVvX1X7JjeJvQgKdwpDcO0BNZ5XEcP1Ds2md\nr7ShPS6EcX+71Ssb7WXUwansqnxioQTqA0ZF5IYBlOsYYvB0A6aJq9wrwmBc\nvLLNiVloICNJ10HDcZ/r7XPdMyXRZHiNAetLedLsDFiqMzsZOGSlMf1gU97A\nOu2B26xE09rEv4HogTX3pmurkjjtVIQijAummHGet/c2QQZgkOUIa8rOj683\nlR3hvGVy4xUZtVsIIibGqgUYKqnhKFGY20A6lXHOWgy390bVK3BEZp3AE1eL\nVirh9R7yf2J3LNgIIbDlxXP3z/o5e11KFxX1CHsNCDpa2+nS7kZDDfNw/YCZ\nHC805oI5RMvB09vDdDCdF6cImactFXmgbFIOc1e9k4GplwgHYgXw+Vqrc3jw\nbbXQGJnQ2SWXBy8+M9/tAQaTRSjS8XnxxHxubqv7M6GV8hUMP7ZexC1G2pIo\nd50q\r\n=QR+v\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIF+OGEvuRJriilsVJkbakU1KyDRLTl9kWhdjgWoIgh/YAiEA1+W9nx4yVoNdyhdGLJzWTA8F5jHVdxvdnEv7HhNCO9k="}]},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_2.0.2_1624372169132_0.740592359712128"},"_hasShrinkwrap":false},"2.0.3":{"name":"robots-txt-parser","version":"2.0.3","description":"A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.","keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"],"main":"src/index.js","directories":{"test":"test"},"scripts":{"test":"mocha test/**/*.js","test-robots":"mocha test/robots/*.js","test-parser":"mocha test/parser/*.js","test-util":"mocha test/util/*.js","coverage":"nyc --reporter=lcov mocha test/**/*.js --timeout 10s","local-coverage":"nyc mocha test/**/*.js"},"repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"license":"MIT","bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser","dependencies":{"fast-url-parser":"^1.1.3","is-absolute-url":"^2.1.0","simple-get":"^4.0.1"},"devDependencies":{"chai":"^4.3.4","eslint-config-airbnb":"^18.2.1","eslint-plugin-import":"^2.23.2","lodash":"^4.17.21","mocha":"^8.3.2","nyc":"^15.1.0"},"gitHead":"9800e12d35cdf0af1230eadb3bce0ff606d07c2f","_id":"robots-txt-parser@2.0.3","_nodeVersion":"16.2.0","_npmVersion":"7.14.0","dist":{"integrity":"sha512-jRezwRFSM/4HdmSKGdtlswEqKnOjSuXA4gMcJgfFfFO6gLZLITdf+8Tt7rDw5pDS6Ey2o5bGYvEl8FcaO4fisA==","shasum":"138c53f7a404fbc70e5f7dccb2a7710d302c862f","tarball":"https://registry.npmjs.org/robots-txt-parser/-/robots-txt-parser-2.0.3.tgz","fileCount":41,"unpackedSize":59628,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDJL/UNQF2QZQuy5/1ldeOEcFs0AR8EyigcBjWyl6TwpQIhALv9a2+oO4jquRkNV+PHg5cBq24xk56dSRzNBHkCG196"}],"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\nwsFzBAEBCAAGBQJidVioACEJED1NWxICdlZqFiEECWMYAoorWMhJKdjhPU1b\r\nEgJ2VmpY5A/9Hqxi5iHRR8JkU1HmNMxIm0qaBDhWUdDLmsPSWMVaU485wOyI\r\nzmaPrl4HUZan4Fb2LLkpbv5V4n+BT46eVR5Q2wP4u5DwDvbumtDfwLnSZM2b\r\nsRzdIKCw/NJNi1x7dPMd62oVB3VQFuH9jXBOFgUYB+P2wNAsFFiI2SVz+bYO\r\npxGdiVjCKMjGfV/6Pvz99HSLDCJmiocAVkumWyJQIB4LuEivvNwbavmsvN36\r\nsG+FF4Tz4wqBjQWUaiBTawpKFwps1q4v9xBeRV9eldka5zJjCEL+V1rmReuM\r\n65trNi6deyaKIN58OMb8bJ+SswhN0wOXOnroekPqtePH6cG5vUNz0jO2Li2G\r\nuEo3WepbOY3SpcTBgunOFC6Mjrplv7cxJKpFeS2R2s7Fx2CPHH4esDnNJRJf\r\n1eP2NaMWjCc94s7wfiFW9fVySM0sixHbU+ijFpQuaaI9MODscSXGMMvWWI0O\r\n4IxHTApbgxjEMOeT1QstMrsvwqy1qoerTTkcXR6YShJZEBRCaRe0d6+PWuy0\r\nKfDTPraNmG8wW1HSin5/odhkDkG5G/rbJu8D7DpbWS//uVZzpEWWaQb1fq/+\r\nTbpgIA9Va5qcsHCJw3QzZCRd/N0RgcUJ5FuBj8ytPFYTf6KRDv/9v1rpU8rM\r\n9/jh16g88BuFFYrG54RF7oOXiJVa+Nxc6V4=\r\n=qb8j\r\n-----END PGP SIGNATURE-----\r\n"},"_npmUser":{"name":"cakroyd","email":"chris@chrisakroyd.com"},"maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parser_2.0.3_1651857576463_0.8096241373294419"},"_hasShrinkwrap":false}},"readme":"# robots-txt-parser\r\n\r\n[![Coverage Status](https://coveralls.io/repos/github/chrisakroyd/robots-txt-parser/badge.svg?branch=master)](https://coveralls.io/github/chrisakroyd/robots-txt-parser?branch=master)\r\n[![npm version](https://badge.fury.io/js/robots-txt-parser.svg)](https://badge.fury.io/js/robots-txt-parser)\r\n![npm](https://img.shields.io/npm/dm/robots-txt-parser)\r\n\r\nA lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.\r\n\r\n## Installing\r\nVia NPM: `npm install robots-txt-parser --save`.\r\n\r\n## Getting Started\r\n\r\nAfter installing robots-txt-parser it needs to be required and initialised:\r\n```js\r\nconst robotsParser = require('robots-txt-parser');\r\nconst robots = robotsParser(\r\n  {\r\n    userAgent: 'Googlebot', // The default user agent to use when looking for allow/disallow rules, if this agent isn't listed in the active robots.txt, we use *.\r\n    allowOnNeutral: false // The value to use when the robots.txt rule's for allow and disallow are balanced on whether a link can be crawled.\r\n  });\r\n```\r\n\r\nExample Usage:\r\n\r\n\r\n```js\r\nconst robotsParser = require('robots-txt-parser');\r\n\r\nconst robots = robotsParser(\r\n  {\r\n    userAgent: 'Googlebot', // The default user agent to use when looking for allow/disallow rules, if this agent isn't listed in the active robots.txt, we use *.\r\n    allowOnNeutral: false, // The value to use when the robots.txt rule's for allow and disallow are balanced on whether a link can be crawled.\r\n  },\r\n);\r\n\r\nrobots.useRobotsFor('http://example.com')\r\n  .then(() => {\r\n    robots.canCrawlSync('http://example.com/news'); // Returns true if the link can be crawled, false if not.\r\n    robots.canCrawl('http://example.com/news', (value) => {\r\n      console.log('Crawlable: ', value);\r\n    }); // Calls the callback with true if the link is crawlable, false if not.\r\n    robots.canCrawl('http://example.com/news') // If no callback is provided, returns a promise which resolves with true if the link is crawlable, false if not.\r\n      .then((value) => {\r\n        console.log('Crawlable: ', value);\r\n      });\r\n  });\r\n```\r\n\r\n## Condensed Documentation\r\n\r\nBelow is a condensed form of the documentation, each is a function that can be found on the robotsParser object.\r\n\r\n| Method | Parameters | Return |\r\n| ----------- | -------- | ------ |\r\n| [parseRobots(key, string)](#parserobots) | `key:String, string:String` | None |\r\n| [isCached(domain)](#iscached) | `domain:String` | Boolean for whether robots.txt for url is cached. |\r\n| [fetch(url)](#fetch) | `url:String` | Promise, resolved when robots.txt retrieved. |\r\n| [useRobotsFor(url)](#userobotsfor) | `url:String` | Promise, resolved when robots.txt is fetched. |\r\n| [canCrawl(url)](#cancrawl)| `url:String, callback:Func (Opt)` | Promise, resolves with Boolean. |\r\n| [getSitemaps()](#getsitemaps)| `callback:Func (Opt)` | Promise if no callback provided, resolves with [String]. |\r\n| [getCrawlDelay()](#getcrawldelay) | `callback:Func (Opt)` | Promise if no callback provided, resolves with Number. |\r\n| [getCrawlableLinks(links)](#getcrawlablelinks) | `links:[String], callback:Func (Opt)` | Promise if no callback provided, resolves with [String]. |\r\n| [getPreferredHost()](#getpreferredhost) | `callback:Func (Opt)` | Promise if no callback provided, resolves with String. |\r\n| [setUserAgent(userAgent)](#setuseragent) | `userAgent:String` | None. |\r\n| [setAllowOnNeutral(allow)](#setallowonneutral)  | `allow:Boolean` | None. |\r\n| [clearCache()](#clearcache) | `None` | None. |\r\n\r\n## Full Documentation\r\n\r\n---\r\n\r\n### parseRobots\r\n`robots.parseRobots(key, string)`\r\nParses a string representation of a robots.txt file and cache's it with the given key.\r\n\r\n##### Parameters\r\n* key -> Can be any URL.\r\n* string -> String representation of a robots.txt file.\r\n\r\n##### Returns\r\nNone.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.parseRobots('https://example.com',\r\n  `\r\n   User-agent: *\r\n   Allow: /*.php$\r\n   Disallow: /\r\n  `);\r\n```\r\n\r\n---\r\n\r\n### isCached\r\n`robots.isCached(domain)`\r\nA method used to check if a robots.txt has already been fetched and parsed.\r\n\r\n##### Parameters\r\n* domain -> Can be any URL.\r\n\r\n##### Returns\r\nReturns true if a robots.txt has already been fetched and cached by the robots-txt-parser.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.isCached('https://example.com'); // true or false\r\nrobots.isCached('example.com'); // Attempts to check the cache for only http:// and returns true or false.\r\n```\r\n\r\n---\r\n\r\n### fetch\r\n`robots.fetch(url)`\r\nAttempts to fetch and parse a robots.txt file located at the url, this method avoids checking the built-in cache and will always attempt to retrieve a fresh copy of the robots.txt.\r\n\r\n##### Parameters\r\n* url -> Any URL.\r\n\r\n##### Returns\r\nReturns a Promise which will resolve once the robots.txt has been fetched with the parsed robots.txt.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.fetch('https://example.com/robots.txt')\r\n    .then((tree) => {\r\n        console.log(Object.keys(tree)); // Will log sitemap and any user agents.\r\n    });\r\n```\r\n\r\n---\r\n\r\n### useRobotsFor\r\n`robots.useRobotsFor(url)`\r\nAttempts to download and use the robots.txt at the given url, if the robots.txt has already been downloaded, reads from the cached copy instead.\r\n\r\n##### Parameters\r\n* url -> Any URL.\r\n\r\n##### Returns\r\nReturns a promsise that resolves once the URL is fetched and parsed.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.useRobotsFor('https://example.com/news')\r\n    .then(() => {\r\n        // Logic to check if links are crawlable.\r\n    });\r\n```\r\n\r\n---\r\n\r\n### canCrawl\r\n`robots.canCrawl(url, callback)`\r\nTests whether a url can be crawled for the current active robots.txt and user agent. If a robots.txt isn't cached for the domain of the url, it is fetched and parsed before returning a boolean value.\r\n##### Parameters\r\n* url -> Any URL.\r\n* callback -> An optional callback, if undefined returns a promise.\r\n\r\n##### Returns\r\nReturns a Promise which will resolve with a boolean value.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.canCrawl('https://example.com/news')\r\n    .then((crawlable) => {\r\n        console.log(crawlable); // Will log a boolean value.\r\n    });\r\n```\r\n\r\n---\r\n\r\n### getSitemaps\r\n`robots.getSitemaps(callback)`\r\nReturns a list of sitemaps present on the active robots.txt.\r\n\r\n##### Parameters\r\n* callback -> An optional callback, if undefined returns a promise.\r\n\r\n##### Returns\r\nReturns a Promise which will resolve with an array of strings.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.getSitemaps()\r\n    .then((sitemaps) => {\r\n        console.log(sitemaps); // Will log an list of strings.\r\n    });\r\n```\r\n\r\n---\r\n\r\n### getCrawlDelay\r\n`robots.getCrawlDelay(callback)`\r\nReturns the crawl delay on requests to the current active robots.txt.\r\n\r\n##### Parameters\r\n* callback -> An optional callback, if undefined returns a promise.\r\n\r\n##### Returns\r\nReturns a Promise which will resolve with an Integer.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.getCrawlDelay()\r\n    .then((crawlDelay) => {\r\n        console.log(crawlDelay); // Will be an Integer greater than or equal to 0.\r\n    });\r\n```\r\n\r\n---\r\n\r\n### getCrawlableLinks\r\n`robots.getCrawlableLinks(links, callback)`\r\nTakes an array of links and returns an array of links which are crawlable\r\nfor the current active robots.txt.\r\n\r\n##### Parameters\r\n* links -> An array of links to check for crawlability.\r\n* callback -> An optional callback, if undefined returns a promise.\r\n\r\n##### Returns\r\nA Promise that will resolve to contain an Array of all the crawlable links.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.getCrawlableLinks([])\r\n    .then((links) => {\r\n        console.log(links);\r\n    });\r\n```\r\n\r\n---\r\n\r\n### getPreferredHost\r\n`robots.getPreferredHost(callback)`\r\nReturns the preferred host name specified in the active robots.txt's host: directive or null if there isn't one.\r\n\r\n##### Parameters\r\n* callback -> An optional callback, if undefined returns a promise.\r\n\r\n##### Returns\r\nAn String if the host is defined, undefined otherwise.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.getPreferredHost()\r\n    .then((host) => {\r\n        console.log(host);\r\n    });\r\n```\r\n\r\n---\r\n\r\n### setUserAgent\r\n`robots.setUserAgent(userAgent)`\r\nSets the current user agent to use when checking if a link can be crawled.\r\n\r\n##### Parameters\r\n* userAgent -> A string.\r\n\r\n##### Returns\r\nundefined\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.setUserAgent('exampleBot'); // When interacting with the robots.txt we now look for records for 'exampleBot'.\r\nrobots.setUserAgent('testBot'); // When interacting with the robots.txt we now look for records for 'testBot'.\r\n```\r\n\r\n---\r\n\r\n### setAllowOnNeutral\r\n`robots.setAllowOnNeutral(allow)`\r\nSets the canCrawl behaviour to return true or false when the robots.txt rules are balanced on whether a link should be crawled or not.\r\n\r\n##### Parameters\r\n* allow -> A boolean value.\r\n\r\n##### Returns\r\nundefined\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.setAllowOnNeutral(true); // If the allow/disallow rules are balanced, canCrawl returns true.\r\nrobots.setAllowOnNeutral(false); // If the allow/disallow rules are balanced, canCrawl returns false.\r\n```\r\n\r\n---\r\n\r\n### clearCache\r\n`robots.clearCache()`\r\nThe cache can get extremely long over extended crawling, this simple method resets the cache.\r\n\r\n##### Parameters\r\nNone\r\n\r\n##### Returns\r\nNone\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.clearCache();\r\n```\r\n\r\n---\r\n\r\n## Synchronous API\r\nSynchronous variants of the API, will be deprecated in a future version.\r\n### canCrawlSync\r\n`robots.canCrawlSync(url)`\r\nTests whether a url can be crawled for the current active robots.txt and user agent. This won't attempt to fetch the robots.txt if it is not cached.\r\n\r\n##### Parameters\r\n* url -> Any url.\r\n\r\n##### Returns\r\nReturns a boolean value depending on whether the url is crawlable. If there is no cached robots.txt for this url, it will always return true.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.canCrawlSync('https://example.com/news') // true or false.\r\n```\r\n\r\n---\r\n\r\n### getSitemapsSync()\r\n`robots.getSitemapsSync()`\r\nReturns a list of sitemaps present on the active robots.txt.\r\n\r\n##### Parameters\r\nNone\r\n##### Returns\r\nAn Array of Strings.\r\n##### Example\r\n\r\n```js\r\nrobots.getSitemapsSync(); // Will be an array e.g. ['http://example.com/sitemap1.xml', 'http://example.com/sitemap2.xml'].\r\n```\r\n\r\n---\r\n\r\n### getCrawlDelaySync()\r\n`robots.getCrawlDelaySync()`\r\nReturns the crawl delay on specified in the active robots.txt's for the active user agent\r\n\r\n##### Parameters\r\nNone\r\n##### Returns\r\nAn Integer greater than or equal to 0.\r\n##### Example\r\n\r\n```js\r\nrobots.getCrawlDelaySync(); // Will be an Integer.\r\n```\r\n\r\n---\r\n\r\n### getCrawlableLinksSync\r\n`robots.getCrawlableLinksSync(links)`\r\nTakes an array of links and returns an array of links which are crawlable\r\nfor the current active robots.txt.\r\n##### Parameters\r\n* links -> An array of links to check for crawlability.\r\n\r\n##### Returns\r\nAn Array of all the links are crawlable.\r\n\r\n##### Example\r\n\r\n```js\r\nrobots.getCrawlableLinks(['example.com/test/news', 'example.com/test/news/article']);  // Will return an array of the links that can be crawled.\r\n```\r\n\r\n---\r\n\r\n### getPreferredHostSync\r\n`robots.getPreferredHostSync()`\r\nReturns the preferred host name specified in the active robots.txt's host: directive or undefined if there isn't one.\r\n\r\n##### Parameters\r\nNone\r\n##### Returns\r\nAn String if the host is defined, undefined otherwise.\r\n##### Example\r\n\r\n```js\r\nrobots.getPreferredHostSync(); // Will be a string if the host directive is defined .\r\n```\r\n\r\n# License\r\nSee [LICENSE](https://github.com/ChristopherAkroyd/robots-txt-parser/blob/master/LICENSE) file.\r\n\r\n# Resources\r\n\r\n  * [Robots.txt Specifications by Google](http://code.google.com/web/controlcrawlindex/docs/robots_txt.html)\r\n  * [A Standard for Robot Exclusion](http://www.robotstxt.org/orig.html)\r\n  * [Robots.txt Specifications by Yandex](https://yandex.com/support/webmaster/controlling-robot/robots-txt.xml)\r\n","maintainers":[{"name":"cakroyd","email":"chris@chrisakroyd.com"}],"time":{"modified":"2022-06-26T13:46:48.882Z","created":"2017-04-30T21:14:30.897Z","0.1.2":"2017-04-30T21:14:30.897Z","0.1.3":"2017-04-30T21:15:53.511Z","1.0.0":"2017-04-30T21:32:27.848Z","1.0.1":"2017-05-01T20:13:18.113Z","1.0.2":"2021-03-30T13:40:32.274Z","1.0.3":"2021-04-10T14:53:23.468Z","1.0.4":"2021-05-13T10:16:14.866Z","1.0.5":"2021-05-16T15:39:20.766Z","1.0.6":"2021-05-16T18:32:36.733Z","1.0.7":"2021-05-16T18:35:27.104Z","1.0.8":"2021-05-21T16:28:07.793Z","1.0.9":"2021-05-23T12:13:43.869Z","1.0.10":"2021-06-02T19:43:52.122Z","2.0.0":"2021-06-21T20:18:41.455Z","2.0.1":"2021-06-22T14:27:46.749Z","2.0.2":"2021-06-22T14:29:29.254Z","2.0.3":"2022-05-06T17:19:36.693Z"},"homepage":"https://github.com/ChrisAkroyd/robots-txt-parser","repository":{"type":"git","url":"git+https://github.com/ChrisAkroyd/robots-txt-parser.git"},"author":{"name":"Chris Akroyd"},"bugs":{"url":"https://github.com/ChrisAkroyd/robots-txt-parser/issues"},"license":"MIT","readmeFilename":"README.md","users":{"cakroyd":true,"keyn":true},"keywords":["robots","txt","robots.txt","parser","crawler","spider","bot","robotstxt","scraper"]}