{"_id":"@attestate/extraction-worker","_rev":"3-36bb5b1d03ae608f2c9febdd421b149f","name":"@attestate/extraction-worker","dist-tags":{"latest":"0.7.4"},"versions":{"0.7.2":{"name":"@attestate/extraction-worker","version":"0.7.2","keywords":["worker","requests","fetch","queue","batch","extraction"],"author":{"url":"https://timdaub.github.io/","name":"Tim Daubenschütz","email":"tim@daubenschuetz.de"},"license":"GPL-3.0-only","_id":"@attestate/extraction-worker@0.7.2","maintainers":[{"name":"timdaub","email":"tim@daubenschuetz.de"}],"homepage":"https://github.com/attestate/extraction-worker#readme","bugs":{"url":"https://github.com/attestate/extraction-worker/issues"},"ava":{"files":["!test/start.mjs"],"require":["dotenv/config"],"nodeArguments":["--loader=esmock","--no-warnings"],"environmentVariables":{"DEBUG":"@attestate*","RPC_HTTP_HOST":"https://cloudflare-eth.com"}},"dist":{"shasum":"5bdcceffcb79894d57a4865b79ed90e6457b36b4","tarball":"https://registry.npmjs.org/@attestate/extraction-worker/-/extraction-worker-0.7.2.tgz","fileCount":24,"integrity":"sha512-nW9iHrs9rp0emp8Pm3AYpIsuaIWWJo7BF4CcXTsPFtmgZFpYL5orGdUjF+iS6381CVrxeksZKWHboaoMTGyw9g==","signatures":[{"sig":"MEYCIQDW9FXpi/z+PmFEo7Lx0ox6TUSR3TB9cC/da0aDViY38QIhAOQAqDiNPqY9FAAv7iawpPwPiG4MdGxH73wiB6Kv+JUo","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102165,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\nwsFzBAEBCAAGBQJkLZjOACEJED1NWxICdlZqFiEECWMYAoorWMhJKdjhPU1b\r\nEgJ2VmpHmQ//VQZzWEl0Bl4kvKNTKKarO6t5MId+RaZTR2LJwOD8G/RVn1I2\r\nAm/nTNe6HIgtNwzB+kHk68lycMdlBFZtGvCb8MRZ1bMEhGZbrCOy1SXMqZ/v\r\n+gthDhtpu8w2DEUd8Sct+Iy9FWJD7SivZi4tnkYrmTGz3Xp9qj0y1Fyko/Ql\r\njZ9DkI77uIP0uzkX5ognCea2dn6A32Ii19Yb+FZyFxKIyuadUeHVBFM/BiwV\r\ndfbAn7Z9xDyLcHflyomV4HrP5j40WUPuhlHr2reOXSuX19e/LvxjXgZPoDGj\r\nvwaO2iNrdHY1AYGFErJ9boLkwD8oaFOvri228JhgWoVPJ28WtXDwcA6vYHlp\r\n1Ks7HQ5e/rvctbBgLCUviYxDQGekY18X2ynA/wQhpqynZ72q2qMzIn1ajzia\r\n4EbsLuSVEBazIkbcn9FiNYMHtFxRD+LVb93AT+XKmQl365xGy5AVvry90YER\r\nM2eOJZSN/9qwis1wfWC1O738f9VlbYxhP9LwcTR3OocYpeQE6YhcJDVMwAsr\r\nTwTJkV7kSzgHjn50/M/+wMDoyfkZtm4tPuqEJuoCgd6xOSEuE/1S6z4orBy0\r\nYAk26vxAGBq4VHv1LqKCw2Ukw+zq6ucYw+mnNtx7Kw9Kamuk9gNRTjY3tlDI\r\nmDeF3gfp5G82zkD/WDECqzpJ8SGNYCDBlPo=\r\n=AUW1\r\n-----END PGP SIGNATURE-----\r\n"},"main":"./src/worker.mjs","gitHead":"fd128a22c727d50acf2e4fc8ab3f28d1ee4f2518","scripts":{"test":"ava --verbose --timeout=20s","prepare":"husky install"},"_npmUser":{"name":"timdaub","email":"tim@daubenschuetz.de"},"repository":{"url":"git+https://github.com/attestate/extraction-worker.git","type":"git"},"_npmVersion":"9.4.0","description":"A component that accepts JSON objects to execute simple or complex extraction maneuvres.","directories":{"test":"test"},"lint-staged":{"*.{mjs,js,css,md}":"prettier --write"},"_nodeVersion":"19.6.0","dependencies":{"ajv":"8.11.0","debug":"4.3.4","fastq":"1.13.0","dotenv":"16.0.0","limiter":"2.0.1","ajv-formats":"2.1.1","cross-fetch":"3.1.5","multiformats":"9.9.0","abort-controller":"3.0.0","@neume-network/schema":"0.8.1"},"_hasShrinkwrap":false,"devDependencies":{"ava":"4.2.0","husky":"7.0.4","esmock":"2.2.1","eth-fun":"0.9.x","prettier":"2.6.2","lint-staged":"12.3.8","expressively-mocked-fetch":"0.2.3"},"peerDependencies":{"eth-fun":"0.9.x"},"_npmOperationalInternal":{"tmp":"tmp/extraction-worker_0.7.2_1680709837882_0.09332826840897579","host":"s3://npm-registry-packages"}},"0.7.3":{"name":"@attestate/extraction-worker","version":"0.7.3","keywords":["worker","requests","fetch","queue","batch","extraction"],"author":{"url":"https://timdaub.github.io/","name":"Tim Daubenschütz","email":"tim@daubenschuetz.de"},"license":"GPL-3.0-only","_id":"@attestate/extraction-worker@0.7.3","maintainers":[{"name":"timdaub","email":"tim@daubenschuetz.de"}],"homepage":"https://github.com/attestate/extraction-worker#readme","bugs":{"url":"https://github.com/attestate/extraction-worker/issues"},"ava":{"files":["!test/start.mjs"],"require":["dotenv/config"],"nodeArguments":["--loader=esmock","--no-warnings"],"environmentVariables":{"DEBUG":"@attestate*","RPC_HTTP_HOST":"https://cloudflare-eth.com"}},"dist":{"shasum":"4302aa8b5b264103f3a29d0d7ae36224890ea20b","tarball":"https://registry.npmjs.org/@attestate/extraction-worker/-/extraction-worker-0.7.3.tgz","fileCount":24,"integrity":"sha512-9c8ai9tbsxuqGp64bwfDp6SMbBlEEe8QWXtsYKL3Mq0IhZC9WyROEBd4BENfi3NvP/V9iBG91MuCdDe1nC5SyQ==","signatures":[{"sig":"MEUCIFH/P8JCn5EUPXx1fhPWfJc1v0k9KhUN04NfH+mZTX0/AiEA2ILMbQ0Rp7XQlkkow6wHEcYTIXz7ms8himq+rMyhLYw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":103006},"main":"./src/worker.mjs","gitHead":"c21b532bf31e50b6ff25ab977f3d40db3de29de2","scripts":{"test":"ava --verbose --timeout=20s","prepare":"husky install"},"_npmUser":{"name":"timdaub","email":"tim@daubenschuetz.de"},"repository":{"url":"git+https://github.com/attestate/extraction-worker.git","type":"git"},"_npmVersion":"6.14.18","description":"A component that accepts JSON objects to execute simple or complex extraction maneuvres.","directories":{"test":"test"},"lint-staged":{"*.{mjs,js,css,md}":"prettier --write"},"_nodeVersion":"14.21.3","dependencies":{"ajv":"8.11.0","debug":"4.3.4","fastq":"1.13.0","dotenv":"16.0.0","limiter":"2.0.1","ajv-formats":"2.1.1","cross-fetch":"3.1.5","multiformats":"9.9.0","abort-controller":"3.0.0","@neume-network/schema":"0.8.1"},"_hasShrinkwrap":false,"devDependencies":{"ava":"4.2.0","husky":"7.0.4","esmock":"2.2.1","eth-fun":"0.9.x","prettier":"2.6.2","lint-staged":"12.3.8","expressively-mocked-fetch":"0.2.3"},"peerDependencies":{"eth-fun":"0.9.x"},"_npmOperationalInternal":{"tmp":"tmp/extraction-worker_0.7.3_1711840390196_0.6149440600262583","host":"s3://npm-registry-packages"}},"0.7.4":{"name":"@attestate/extraction-worker","version":"0.7.4","description":"A component that accepts JSON objects to execute simple or complex extraction maneuvres.","main":"./src/worker.mjs","ava":{"files":["!test/start.mjs"],"nodeArguments":["--loader=esmock","--no-warnings"],"environmentVariables":{"DEBUG":"@attestate*","RPC_HTTP_HOST":"https://cloudflare-eth.com"},"require":["dotenv/config"]},"lint-staged":{"*.{mjs,js,css,md}":"prettier --write"},"directories":{"test":"test"},"scripts":{"test":"ava --verbose --timeout=20s","prepare":"husky install"},"repository":{"type":"git","url":"git+https://github.com/attestate/extraction-worker.git"},"keywords":["worker","requests","fetch","queue","batch","extraction"],"author":{"name":"Tim Daubenschütz","email":"tim@daubenschuetz.de","url":"https://timdaub.github.io/"},"license":"GPL-3.0-only","bugs":{"url":"https://github.com/attestate/extraction-worker/issues"},"homepage":"https://github.com/attestate/extraction-worker#readme","devDependencies":{"ava":"4.2.0","esmock":"2.7.1","eth-fun":"0.10.x","expressively-mocked-fetch":"0.2.3","husky":"7.0.4","lint-staged":"12.3.8","prettier":"2.6.2"},"dependencies":{"@neume-network/schema":"0.8.1","abort-controller":"3.0.0","ajv":"8.11.0","ajv-formats":"2.1.1","cross-fetch":"3.1.5","debug":"4.3.4","dotenv":"16.0.0","fastq":"1.13.0","limiter":"2.0.1","multiformats":"9.9.0"},"peerDependencies":{"eth-fun":"0.10.x"},"_id":"@attestate/extraction-worker@0.7.4","gitHead":"d74cc97bbf79d5e75d530727e9e481496dd9bb68","_nodeVersion":"20.19.4","_npmVersion":"10.8.2","dist":{"integrity":"sha512-013TKDWmGxJRuz4s/G3MKIL/xGgRZmcWhAFee4zkzbptEwKCD/Ek6ECBYVmdfXOsnYpSQjUP3eLbtqTZ58nduA==","shasum":"e9c787bd8124114499cc7d38d8abfc932009aeb7","tarball":"https://registry.npmjs.org/@attestate/extraction-worker/-/extraction-worker-0.7.4.tgz","fileCount":24,"unpackedSize":103800,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIEJ6uYTz1IDg9219OZifOc3J0oOYdqTvZTDB5HEhW0hHAiEApPJbkSdfQ0ZK1n200/0gig6W0vJCzxAiczff/XFxg7E="}]},"_npmUser":{"name":"timdaub","email":"tim@daubenschuetz.de"},"maintainers":[{"name":"timdaub","email":"tim@daubenschuetz.de"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/extraction-worker_0.7.4_1754902203059_0.11789980550518853"},"_hasShrinkwrap":false}},"time":{"created":"2023-04-05T15:50:37.816Z","modified":"2025-08-11T08:50:03.539Z","0.7.2":"2023-04-05T15:50:38.160Z","0.7.3":"2024-03-30T23:13:10.337Z","0.7.4":"2025-08-11T08:50:03.350Z"},"bugs":{"url":"https://github.com/attestate/extraction-worker/issues"},"author":{"name":"Tim Daubenschütz","email":"tim@daubenschuetz.de","url":"https://timdaub.github.io/"},"license":"GPL-3.0-only","homepage":"https://github.com/attestate/extraction-worker#readme","keywords":["worker","requests","fetch","queue","batch","extraction"],"repository":{"type":"git","url":"git+https://github.com/attestate/extraction-worker.git"},"description":"A component that accepts JSON objects to execute simple or complex extraction maneuvres.","maintainers":[{"name":"timdaub","email":"tim@daubenschuetz.de"}],"readme":"# @attestate/extraction-worker\n\nThe extraction-worker is a component that accepts JSON objects to execute\nsimple or complex extraction maneuvres. Originally, the extraction worker was\nconceived as utility tool for `@neume-network/extraction-worker`, but it is\nfrankly useful for any kind of task that requires downloading a bigger chunk of\ndata from various endpoints.\n\nThe extraction worker supports: JSON-RPC, GraphQL, HTTPS, Arweave and IPFS.\n\n## Installation\n\n```bash\nnpm i @attestate/extraction-worker\nnpm i eth-fun --no-save # to install eth-fun as a peer dependency\n```\n\n## Usage\n\nIn general, also consider looking at the [crawler's main documentation page](https://attestate.com/crawler/main/configuration.html#configuration-file)\nto find out more about extraction-worker configuration options.\n\nIf you're looking to extract a single message, use the `execute` function.\n\n```js\nimport { execute } from \"@attestate/extraction-worker\";\n\nconst message = {\n  version: \"0.0.1\",\n  type: \"json-rpc\",\n  method: \"eth_getTransactionReceipt\",\n  params: [\n    \"0xed14c3386aea0c5b39ffea466997ff13606eaedf03fe7f431326531f35809d1d\",\n  ],\n  options: {\n    url: \"https://...\"\n  }\n};\n\nconst outcome = await execute(message);\n\nif (!outcome.results) {\n  console.error(outcome.error);\n  return;\n}\nconsole.log(outcome.results);\n```\n\nElse, you can use the extraction worker to stream tasks. For that, you'll need\nto create a worker execution module\n\nworker_start.mjs\n\n```js\nimport \"dotenv/config\";\nimport { Worker, isMainThread, workerData } from \"worker_threads\";\n\nimport logger from \"./logger.mjs\";\nimport { run } from \"@attestate/extraction-worker\";\n\nconst log = logger(\"start\");\nconst module = {\n  defaults: {\n    workerData: {\n      queue: {\n        options: {\n          concurrent: 1,\n        },\n      },\n    },\n  },\n};\n\nif (isMainThread) {\n  log(\"Detected mainthread: Respawning extractor as worker_thread\");\n  // INFO: We're launching this file as a `Worker` when the mainthread is\n  // detected as this can be useful when running it without an accompanying\n  // other process.\n  new Worker(__filename, { workerData: module.defaults.workerData });\n} else {\n  run();\n}\n```\n\nYou can then execute it as follows\n\n```js\nimport { once } from \"events\";\nimport { Worker } from \"worker_threads\";\n\nconst worker = new Worker(workerPath, {\n  workerData: {\n    queue: {\n      options: {\n        concurrent: 1,\n      },\n    },\n  },\n});\n\nconst message = {\n  version: \"0.0.1\",\n  type: \"json-rpc\",\n  method: \"eth_getTransactionReceipt\",\n  params: [\n    \"0xed14c3386aea0c5b39ffea466997ff13606eaedf03fe7f431326531f35809d1d\",\n  ],\n  options: {\n    url: \"https://...\"\n  }\n};\nworker.postMessage(message);\nconst [outcome] = await once(w, \"message\");\n\nif (!outcome.results) {\n  console.error(outcome.error);\n  return;\n}\nconsole.log(outcome.results);\n```\n\n## License\n\nGPL-3.0-only, see LICENSE file for details.\n","readmeFilename":"readme.md"}