{"_id":"robots-txt-parse","_rev":"30-228f1922bdff461472665827f49cdc74","name":"robots-txt-parse","description":"Streaming parser for robots.txt files","dist-tags":{"latest":"2.0.1"},"versions":{"0.0.1":{"name":"robots-txt-parse","version":"0.0.1","description":"Streaming parser for robots.txt files","main":"lib/parse.js","scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"chai":"^1.9.1","mocha":"^1.18.2"},"dependencies":{"bluebird":"^2.3.5","split":"^0.3.0","stream-combiner":"^0.2.1","through":"^2.3.4"},"gitHead":"665ba4de879f9cf32b4c5166cd8ab5c65dd453dc","_id":"robots-txt-parse@0.0.1","_shasum":"b61eac21eb21914263382634311b2f33bd095fee","_from":".","_npmVersion":"1.4.28","_npmUser":{"name":"janpotoms","email":"potoms.jan@gmail.com"},"maintainers":[{"name":"janpotoms","email":"potoms.jan@gmail.com"}],"dist":{"shasum":"b61eac21eb21914263382634311b2f33bd095fee","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-0.0.1.tgz","integrity":"sha512-EgsSwjMniKbra927oU3GIylitt15SJq3gYVqRTExisI6eCtvAUl7snFRB7wPIdL1o8s/lu8GX98CsmjDRFseJQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIAy4abALNl2ohlbtCSOC7FFkClrSznMLUH6bwp+mcQZ/AiBnEjtkTHwRMIrjSaHgvJtTQBqVfucJDQtRAEbGsVegjg=="}]},"directories":{}},"0.0.2":{"name":"robots-txt-parse","version":"0.0.2","description":"Streaming parser for robots.txt files","main":"lib/parse.js","scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"chai":"^1.9.1","mocha":"^1.18.2"},"dependencies":{"bluebird":"^2.3.5","split":"^0.3.0","stream-combiner":"^0.2.1","through":"^2.3.4"},"gitHead":"202b728ba38efcf954846c15cf2e7646c683a5ad","_id":"robots-txt-parse@0.0.2","_shasum":"075c2bb7a148f0df5b7e1f5d9edcfbe4f8f1d6bf","_from":".","_npmVersion":"1.4.28","_npmUser":{"name":"janpotoms","email":"potoms.jan@gmail.com"},"maintainers":[{"name":"janpotoms","email":"potoms.jan@gmail.com"}],"dist":{"shasum":"075c2bb7a148f0df5b7e1f5d9edcfbe4f8f1d6bf","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-0.0.2.tgz","integrity":"sha512-l8BYDJxpzNgsNgStOt3Qi2qXYFBF4Yb6yzivOKHQTps0zIYvR7veVmDrNvLENuwPPYMiZh6NE12uUL4dnMhokQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIACwPCgMfkQTYRG2fFIBRDTswga4RknsjayPBdyv6XG0AiEA3U+SmRIbE9Dy/EfnfJ8AVBJd3qWmfZZ1NPLhDNdig20="}]},"directories":{}},"0.0.3":{"name":"robots-txt-parse","version":"0.0.3","description":"Streaming parser for robots.txt files","main":"lib/parse.js","scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"chai":"^1.9.1","mocha":"^1.18.2"},"dependencies":{"bluebird":"^2.3.5","split":"^0.3.0","stream-combiner":"^0.2.1","through":"^2.3.4"},"gitHead":"7c2fbb2025eb5df2c6c5c7f253b4c96f3e20ab1c","_id":"robots-txt-parse@0.0.3","_shasum":"71a08cdec0717896e49829202e089e103e150921","_from":".","_npmVersion":"1.4.28","_npmUser":{"name":"janpotoms","email":"potoms.jan@gmail.com"},"maintainers":[{"name":"janpotoms","email":"potoms.jan@gmail.com"}],"dist":{"shasum":"71a08cdec0717896e49829202e089e103e150921","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-0.0.3.tgz","integrity":"sha512-4CuVezLrVtWmonWjy+TXXQ+aqlG/5E2kwaiUD6c6ccCHprsOYyOVgNNdeVlSojhacBiRI2+W5za2E8D/GJvKXg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHhtqL2+wAsucwldC4KR5yMkpF5bkSusMATQZQZN4/8BAiBAyU2DegEKFZkuBEt0l6lRT/pVmtRV5TQN4zeMlOUqtQ=="}]},"directories":{}},"0.0.4":{"name":"robots-txt-parse","version":"0.0.4","description":"Streaming parser for robots.txt files","main":"lib/parse.js","scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"chai":"^1.9.1","mocha":"^1.18.2"},"dependencies":{"bluebird":"^2.3.5","split":"^0.3.0","stream-combiner":"^0.2.1","through":"^2.3.4"},"gitHead":"c8b0683233928973d4abfb0d7085beaa6984e661","_id":"robots-txt-parse@0.0.4","_shasum":"f7d1f323f79921d7e9c6c4bbd25048f6e9810d71","_from":".","_npmVersion":"3.3.6","_nodeVersion":"5.0.0","_npmUser":{"name":"janpotoms","email":"potoms.jan@gmail.com"},"dist":{"shasum":"f7d1f323f79921d7e9c6c4bbd25048f6e9810d71","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-0.0.4.tgz","integrity":"sha512-B2VQEC5oe9MD/77oILGi98NNAqtY1BlqgrwlYjq/RR5sDfZqpkFj1sG5pCieiSApoTxatJgGW+yWCh0zFfOGMQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIFWLlER5i/1Nw23+ouSg+BZNuRmdB9za8taBg7XkH1RjAiBiuh4RyEMlJmNoMYkeO6vibhVqYFf6bG7qZqCMBNa/NA=="}]},"maintainers":[{"name":"janpotoms","email":"potoms.jan@gmail.com"}],"directories":{}},"1.0.0":{"name":"robots-txt-parse","version":"1.0.0","description":"Streaming parser for robots.txt files","main":"lib/parse.js","repository":{"type":"git","url":"git+https://github.com/Woorank/robots-txt-parse.git"},"scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test","version":"echo $npm_package_version"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"chai":"^1.9.1","mocha":"^1.18.2"},"dependencies":{"bluebird":"^2.3.5","split":"^0.3.0","stream-combiner":"^0.2.1","through":"^2.3.4"},"gitHead":"16d4c5e6a14a137fd1e0bbeb0b348106deaa35dd","bugs":{"url":"https://github.com/Woorank/robots-txt-parse/issues"},"homepage":"https://github.com/Woorank/robots-txt-parse#readme","_id":"robots-txt-parse@1.0.0","_npmVersion":"5.5.1","_nodeVersion":"8.9.1","_npmUser":{"name":"woorank-ci","email":"dev+npm-ci@woorank.com"},"dist":{"integrity":"sha512-vN9MCjo9Qx/gWTGFVYTwlIG6IpPgX4m55Pm9HFuojaV1wzHArbNay8kW+ZDNcT8+FJPnJQoFXZYuqvhOmYfyGg==","shasum":"e96cd729dd8ad7aa097b00e909ccde427b6d2b52","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-1.0.0.tgz","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQD8/5pF80wb638yCZqV2eg0PFt15E/gmGRwtO2+1v0dDAIgZwTiVnaIdpoNjhD4FFuPgFJLlz59DEAsulmhrSzHnLY="}]},"maintainers":[{"email":"potoms.jan@gmail.com","name":"janpotoms"},{"email":"dev+npm-ci@woorank.com","name":"woorank-ci"},{"email":"nils.de.moor@gmail.com","name":"ndemoor"},{"email":"mickael@woorank.com","name":"mickael-van-der-beek"},{"email":"dev+npm@woorank.com","name":"woorank-admin"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parse-1.0.0.tgz_1511969518060_0.1745105453301221"},"directories":{}},"1.0.1":{"name":"robots-txt-parse","version":"1.0.1","description":"Streaming parser for robots.txt files","main":"lib/parse.js","repository":{"type":"git","url":"git+https://github.com/Woorank/robots-txt-parse.git"},"scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test","version":"echo $npm_package_version"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"chai":"^1.9.1","mocha":"^1.18.2"},"dependencies":{"bluebird":"^2.3.5","split":"^0.3.0","stream-combiner":"^0.2.1","through":"^2.3.4"},"gitHead":"5ae45e56504c0e50adbd968858d7a700d89d6584","bugs":{"url":"https://github.com/Woorank/robots-txt-parse/issues"},"homepage":"https://github.com/Woorank/robots-txt-parse#readme","_id":"robots-txt-parse@1.0.1","_npmVersion":"5.5.1","_nodeVersion":"8.9.1","_npmUser":{"name":"woorank-ci","email":"dev+npm-ci@woorank.com"},"dist":{"integrity":"sha512-wYpRH5DNw2f6+p5bZuHpJXO8ZSPmasVUkemQ7S2h8B5wri99LCdJ/T+tJIL2uG+fb5SASO6yA7viDS4xA4vGgQ==","shasum":"452d8e4fc016b739a330bec61998dafe00079f70","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-1.0.1.tgz","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCEOEskw1jYtQF+E8jJzPY0k/CAUwMKaVDuaX4mJF3pCwIgOQq15o9Zr4PZUAJOJj2gboR2LZ26TouXPPxPFNtolWI="}]},"maintainers":[{"email":"potoms.jan@gmail.com","name":"janpotoms"},{"email":"dev+npm-ci@woorank.com","name":"woorank-ci"},{"email":"nils.de.moor@gmail.com","name":"ndemoor"},{"email":"mickael@woorank.com","name":"mickael-van-der-beek"},{"email":"dev+npm@woorank.com","name":"woorank-admin"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parse-1.0.1.tgz_1512394297235_0.3772934917360544"},"directories":{}},"2.0.0":{"name":"robots-txt-parse","version":"2.0.0","description":"Streaming parser for robots.txt files","main":"lib/parse.js","repository":{"type":"git","url":"git+https://github.com/Woorank/robots-txt-parse.git"},"scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test","version":"echo $npm_package_version","test:ci":"npm run lint && npm run typescript && npm run test","typescript":"tsc","lint":"semistandard","fix":"semistandard --fix"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"@types/node":"^14.14.31","@types/split":"^1.0.0","@types/through":"0.0.30","chai":"^4.3.0","mocha":"^8.3.0","semistandard":"^16.0.0","typescript":"^4.1.5"},"dependencies":{"split":"^1.0.1"},"gitHead":"1a1ef45a209c0ec5748f4b14b5910b6c4472c875","bugs":{"url":"https://github.com/Woorank/robots-txt-parse/issues"},"homepage":"https://github.com/Woorank/robots-txt-parse#readme","_id":"robots-txt-parse@2.0.0","_nodeVersion":"10.23.3","_npmVersion":"6.14.11","dist":{"integrity":"sha512-uyxXCyqrsFomLNyFpsRHZY81OsdsIUr2XdkQH5Qm818obm1JO65MoyFQt0EIDkeeb4UoWUYO8te6HXL1C2YI/g==","shasum":"61c95eca0fd546955bbb14e7b164cef1004a6a36","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-2.0.0.tgz","fileCount":17,"unpackedSize":12214,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgNUT5CRA9TVsSAnZWagAA2csP/R+mNliiEfxvKdu7T4Hq\ncow7oocfGVZahtsnHYH2KQsdcWX9SfQb+UUtuKakWqMMXjqBKYBOwPAp70cy\nryotsy78h39XOpeTBoUeGlWiiM38kRXtT4kP7JJ8gaf38zIbX0B4KBZDZKGw\nWFJj2MEqRUDdThalOA6EnguiF0+fqnWQ1VV670HioIr0gUNneItQ46iCzWfa\niqpUDPjpHBiqCBwmK1+Hs6gSG1c4Sy7IM85FzaaVPselwdm0va2JHm1tIZpp\nkjo/IolxX4U/LGmvHgJGPp9HXiAHWU8TeydVXFa9uicQvWJtYyUrYlZKx9qN\nq9AqiswLVTzjFBHfOnBxFWbzwt8sX1uFFYU2irDNHrhNZr1apqGdNMjYaX2R\na3xE7c/bD9NXZ8k6bJIjeRNIHyJ74kNDLAaRWDCZk7cPzVQNPbF/H64B99Td\nDcn8A1LAlJf33tRj4sUP3c3leDms3kkY7JzMi5foyR6StopdXzhLkxe6RIob\nL2x6K4D4TEFzuoNTiqizrz5Mict3PbRM9s5CYquaeOI0scrpViJL4nOkCgMr\nRwsxBZcLUdriD8sKd5xHiFf62nQazQf1dvqVf4XqtEK5gZ0ivl3WXab8mnHi\n13s/IGbAflk/59WmGv+75Xf3NAY4dWMb23zhv2uL40Ct5R1SNS0tfIFZ+oEm\niHl6\r\n=ih3M\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDMgr7dSGnC9E4KmP3yDT2jrTgZ0tDexV8jgR5W3Gx07AIhAOHD018qjwB5eHiqp+WRP6dvxbtxyIjkepY/sQaLmGHx"}]},"_npmUser":{"name":"woorank-ci","email":"dev+npm-ci@woorank.com"},"directories":{},"maintainers":[{"name":"woorank-admin","email":"dev+npm@woorank.com"},{"name":"janpotoms","email":"potoms.jan@gmail.com"},{"name":"ndemoor","email":"nils.de.moor@gmail.com"},{"name":"woorank-ci","email":"dev+npm-ci@woorank.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parse_2.0.0_1614103801041_0.22350176834466606"},"_hasShrinkwrap":false},"2.0.1":{"name":"robots-txt-parse","version":"2.0.1","description":"Streaming parser for robots.txt files","main":"lib/parse.js","repository":{"type":"git","url":"git+https://github.com/Woorank/robots-txt-parse.git"},"scripts":{"test":"mocha -R spec ./test","test-watch":"mocha -w -R spec ./test","version":"echo $npm_package_version","test:ci":"npm run lint && npm run typescript && npm run test","typescript":"tsc","lint":"semistandard","fix":"semistandard --fix"},"author":{"name":"Jan Potoms"},"license":"MIT","devDependencies":{"@types/split2":"^2.1.6","chai":"^4.3.0","mocha":"^8.3.0","semistandard":"^16.0.0","typescript":"^4.1.5"},"dependencies":{"split2":"^3.2.2"},"gitHead":"323dc95e1649171b17e9352f2cbc5340a2264233","bugs":{"url":"https://github.com/Woorank/robots-txt-parse/issues"},"homepage":"https://github.com/Woorank/robots-txt-parse#readme","_id":"robots-txt-parse@2.0.1","_nodeVersion":"10.23.3","_npmVersion":"6.14.11","dist":{"integrity":"sha512-JsD14cdmoch/97AbcgkeIKC5EcfsZagrSFIx5BCV8rR0b1DBCfmIwLQeLxIxvInh1z1Q4TLlfLiPZ8OHqW7FzQ==","shasum":"3633445bbf0db626a48983e0c85561cec7dbd997","tarball":"https://registry.npmjs.org/robots-txt-parse/-/robots-txt-parse-2.0.1.tgz","fileCount":17,"unpackedSize":12194,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgNXx1CRA9TVsSAnZWagAAwBkP/2U/FkXDfmjhz6KFwN3q\nn12OW448gVk4hjCzgG2A5DX5gFEEY0DRGyf1uFpD18Lo3Z+TfcxMlg99DgT8\nliYP9I0O9n/aL0rzAeZ/qB+rcvzBXEIxgMvrWImH+h0nIZBtQz1qt5vdjcIG\nrzONmvUvTFLz4E5I8bguyO8s+/7i9OSFhlRfSzjfNQagZCTy1tvtL06bf6f8\nR6n5FIVHhgp62lB4FT6Z+KTazjSleoZMZ/91vCOhXNat2LxlSD1jm/ucs3zY\nbUjH/Au3UkRmX2FJzvAHIPpDAXd1bk8zLXYQV1MBDayMH0U3ipZKviUQQWgd\n8nptGI26dwe+UP2ZImZkcB+eBdiIPXRdm7L2fEzOkByp+863GHVQ9D1wvrLt\nQR706EDEow2KSB0oa4wrn64QrwQozBSFbKfWHKQa9fE+44i8KBsdAZ3eBt7I\nIDhorzzkABSe+tyH7IIk27z+1T5+tv2rV7/4ksnixpApAuOgGfMvZQKFg0mC\nlytJo3WEbDbmCTQeIFWV7m5Ui3eyUZNhrHx18AWy3IL3xlAGEpxSXYbzGqup\nVxwzZgSLy2NemkpdmKF/3bOFzdFmOsU9epqXcZQEXHVyhFMXcqmKv6UhYRFh\n1N3+5AaDHF3ghbDogWdN6nVmEskrAkjDVLmzXXoJTAouIhA2G5FhHfK1bI2E\n/Pjy\r\n=kTjc\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQC9qicYcHgFTz/rQ3RX554p0bx11wsRmcDlZ3a2YkwFEQIhAMCp1HkJClqv/RfsyoDsdfJaVJwKL3q1Ajen07+dMR7R"}]},"_npmUser":{"name":"woorank-ci","email":"dev+npm-ci@woorank.com"},"directories":{},"maintainers":[{"name":"woorank-admin","email":"dev+npm@woorank.com"},{"name":"janpotoms","email":"potoms.jan@gmail.com"},{"name":"ndemoor","email":"nils.de.moor@gmail.com"},{"name":"woorank-ci","email":"dev+npm-ci@woorank.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/robots-txt-parse_2.0.1_1614118004794_0.4200258638093224"},"_hasShrinkwrap":false}},"readme":"# robots-txt-parse [![Build Status](https://travis-ci.org/Woorank/robots-txt-parse.svg)](https://travis-ci.org/Woorank/robots-txt-parse)\n\nStreaming robots.txt parser\n\n## usage\n\n```js\nconst parse = require('robots-txt-parse');\nconst fs = require('fs');\n\nconst input = fs.createReadStream(__dirname + '/robots.txt');\nconst result = await parse(input);\n```\nassuming this file\n```\nuser-agent: *\nuser-agent: googlebot\ndisallow: /\n\nuser-agent: twitterbot\ndisallow: /\nallow: /twitter\n\nuser-agent: mozilla\ndisallow: /path\nnoindex: /path\n\nSitemap: http://www.example.com/sitemap.xml\n```\nproduces following output\n```json\n{\n  \"groups\": [{\n    \"agents\": [ \"*\", \"googlebot\" ],\n    \"rules\": [\n      { \"rule\": \"disallow\", \"path\": \"/\" }\n    ]\n  }, {\n    \"agents\": [ \"twitterbot\" ],\n    \"rules\": [\n      { \"rule\": \"disallow\", \"path\": \"/\" },\n      { \"rule\": \"allow\", \"path\": \"/twitter\" }\n    ]\n  }, {\n    \"agents\": [ \"mozilla\" ],\n    \"rules\": [\n      { \"rule\": \"disallow\", \"path\": \"/path\" },\n      { \"rule\": \"noindex\", \"path\": \"/path\" }\n    ]\n  }],\n  \"extensions\": [\n    { \"extension\": \"sitemap\", \"value\": \"http://www.example.com/sitemap.xml\" }\n  ]\n}\n```\n","maintainers":[{"name":"woorank-admin","email":"dev+npm@woorank.com"},{"name":"janpotoms","email":"potoms.jan@gmail.com"},{"name":"ndemoor","email":"nils.de.moor@gmail.com"},{"name":"woorank-ci","email":"dev+npm-ci@woorank.com"}],"time":{"modified":"2022-06-26T13:46:47.889Z","created":"2014-10-14T15:59:58.903Z","0.0.1":"2014-10-14T15:59:58.903Z","0.0.2":"2014-10-17T10:10:30.063Z","0.0.3":"2015-03-31T19:09:07.138Z","0.0.4":"2015-12-04T20:38:59.150Z","1.0.0":"2017-11-29T15:31:58.130Z","1.0.1":"2017-12-04T13:31:38.150Z","2.0.0":"2021-02-23T18:10:01.155Z","2.0.1":"2021-02-23T22:06:44.907Z"},"author":{"name":"Jan Potoms"},"license":"MIT","readmeFilename":"README.md","users":{"meganekick":true},"homepage":"https://github.com/Woorank/robots-txt-parse#readme","repository":{"type":"git","url":"git+https://github.com/Woorank/robots-txt-parse.git"},"bugs":{"url":"https://github.com/Woorank/robots-txt-parse/issues"}}