{"_id":"@arcus-xyz/da-reader","_rev":"3-4fb964cfa4f4eb9b361044b3e939d1e8","name":"@arcus-xyz/da-reader","dist-tags":{"latest":"0.2.0"},"versions":{"0.1.0":{"name":"@arcus-xyz/da-reader","version":"0.1.0","keywords":["perps","data-availability","sbe","s3","indexer"],"license":"Apache-2.0","_id":"@arcus-xyz/da-reader@0.1.0","maintainers":[{"name":"k-dydx","email":"ken@dydx.exchange"},{"name":"luka-arcus","email":"luka@dydx.exchange"}],"homepage":"https://github.com/arcus-xyz/da-reader#readme","bugs":{"url":"https://github.com/arcus-xyz/da-reader/issues"},"dist":{"shasum":"557595f8b2bd01a8392d4dbc14bf9f91948e0d9c","tarball":"https://registry.npmjs.org/@arcus-xyz/da-reader/-/da-reader-0.1.0.tgz","fileCount":6,"integrity":"sha512-8Vz5/eLRh8oq9HmR7nUg3puip06xg18iW47337gxe7GLc7CtIPzlU8ds2C3wRc0/sws8Vo1eCl+FrXkkeGFyeA==","signatures":[{"sig":"MEUCIQDTqrfHmH7Tvfjw6rsF7ru71NnPyGyU86YzSfkIQ5BJtQIgW4xQkHsruwk0zmzSEoXL8ild1zaD2c4PsD835elXkCc=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@arcus-xyz%2fda-reader@0.1.0","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"unpackedSize":165422},"type":"module","types":"./dist/index.d.ts","engines":{"node":">=22.12"},"exports":{".":{"types":"./dist/index.d.ts","default":"./dist/index.js"}},"gitHead":"942a0ad69af719751895148190192e590c177075","scripts":{"test":"node --import tsx --test tests/*.test.ts","build":"tsup","prepare":"tsup","typecheck":"tsc --noEmit","prepublishOnly":"npm run typecheck && npm run test && npm run build"},"_npmUser":{"name":"k-dydx","email":"ken@dydx.exchange"},"repository":{"url":"git+https://github.com/arcus-xyz/da-reader.git","type":"git"},"_npmVersion":"10.9.8","description":"Read and decode perps-chain Data Availability records (S3 block-inputs): SigV4 fetch, SBE fill decode, market units, real-time tip following.","directories":{},"sideEffects":false,"_nodeVersion":"22.23.1","publishConfig":{"access":"public","provenance":true},"_hasShrinkwrap":false,"devDependencies":{"tsx":"4.22.4","tsup":"8.5.0","publint":"^0.3.22","typescript":"5.9.3","@types/node":"22.20.0","@arethetypeswrong/cli":"^0.18.5"},"_npmOperationalInternal":{"tmp":"tmp/da-reader_0.1.0_1785971773885_0.9248399129168607","host":"s3://npm-registry-packages-npm-production"}},"0.1.1":{"name":"@arcus-xyz/da-reader","version":"0.1.1","keywords":["perps","data-availability","sbe","s3","indexer"],"license":"Apache-2.0","_id":"@arcus-xyz/da-reader@0.1.1","maintainers":[{"name":"k-dydx","email":"ken@dydx.exchange"},{"name":"luka-arcus","email":"luka@dydx.exchange"}],"homepage":"https://github.com/arcus-xyz/da-reader#readme","bugs":{"url":"https://github.com/arcus-xyz/da-reader/issues"},"dist":{"shasum":"4061429ba19efa30f51e841f08a6a1c642301970","tarball":"https://registry.npmjs.org/@arcus-xyz/da-reader/-/da-reader-0.1.1.tgz","fileCount":6,"integrity":"sha512-pK5MCqByNW80/wwNpMjCa6qZnS1Yo6uHHHrQO52bhOd7V3tizi7yXcMPKyVfCIqkRTw+rL6kwJzZ+8qpGPKETg==","signatures":[{"sig":"MEQCICEWPDWB4beRfFlwup0DJgdwVoB+aKHNHibw2HoHK5TGAiAvXyWGMAHWIvLR9QLboBCTgw8qGgiOIEVv9bJQBDuuMQ==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@arcus-xyz%2fda-reader@0.1.1","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"unpackedSize":165422},"type":"module","types":"./dist/index.d.ts","engines":{"node":">=22.12"},"exports":{".":{"types":"./dist/index.d.ts","default":"./dist/index.js"}},"gitHead":"61d51c6f16b4236300588d1ecc5d77523c759e20","scripts":{"test":"node --import tsx --test tests/*.test.ts","build":"tsup","prepare":"tsup","typecheck":"tsc --noEmit","prepublishOnly":"npm run typecheck && npm run test && npm run build"},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:a77e6feb-8603-4044-b2de-623c30234110"}},"repository":{"url":"git+https://github.com/arcus-xyz/da-reader.git","type":"git"},"_npmVersion":"12.0.2","description":"Read and decode perps-chain Data Availability records (S3 block-inputs): SigV4 fetch, SBE fill decode, market units, real-time tip following.","directories":{},"sideEffects":false,"_nodeVersion":"22.23.1","publishConfig":{"access":"public","provenance":true},"_hasShrinkwrap":false,"devDependencies":{"tsx":"4.22.4","tsup":"8.5.0","publint":"^0.3.22","typescript":"5.9.3","@types/node":"22.20.0","@arethetypeswrong/cli":"^0.18.5"},"_npmOperationalInternal":{"tmp":"tmp/da-reader_0.1.1_1785973125832_0.8777367907214753","host":"s3://npm-registry-packages-npm-production"}},"0.2.0":{"name":"@arcus-xyz/da-reader","version":"0.2.0","description":"Read and decode perps-chain Data Availability records (S3 block-inputs): SigV4 fetch, SBE fill decode, market units, real-time tip following.","license":"Apache-2.0","type":"module","engines":{"node":">=22.12"},"types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","default":"./dist/index.js"}},"sideEffects":false,"publishConfig":{"access":"public","provenance":true},"repository":{"type":"git","url":"git+https://github.com/arcus-xyz/da-reader.git"},"keywords":["perps","data-availability","sbe","s3","indexer"],"scripts":{"build":"tsup","typecheck":"tsc --noEmit","test":"node --import tsx --test tests/*.test.ts","prepare":"tsup","prepublishOnly":"npm run typecheck && npm run test && npm run build"},"devDependencies":{"@arethetypeswrong/cli":"^0.18.5","@types/node":"22.20.0","publint":"^0.3.22","tsup":"8.5.0","tsx":"4.22.4","typescript":"5.9.3"},"gitHead":"ab6b6392d9197cdcbfa12e04702dc05e8c0282f0","_id":"@arcus-xyz/da-reader@0.2.0","bugs":{"url":"https://github.com/arcus-xyz/da-reader/issues"},"homepage":"https://github.com/arcus-xyz/da-reader#readme","_nodeVersion":"22.23.1","_npmVersion":"12.0.2","dist":{"integrity":"sha512-FqS3k5CquUO7q5hF8V5WTlve7KgkLpMLGOlaJQlUiJO5XIhgmmditBte+W443LNiVCJJONr+N6vUmuVAaMozbQ==","shasum":"71ea1ca6fad902c894124061592ab442ced0958d","tarball":"https://registry.npmjs.org/@arcus-xyz/da-reader/-/da-reader-0.2.0.tgz","fileCount":6,"unpackedSize":202469,"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@arcus-xyz%2fda-reader@0.2.0","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEQCIQCHzCjLfBIG6pqR+nIkcXf1InVynZlmqsIl7pCxfYwdjAIfFAWRfBqeq5amcyHISUtNCPeMsAer4BOoSdh55/CAOg=="}]},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:a77e6feb-8603-4044-b2de-623c30234110"}},"directories":{},"maintainers":[{"name":"k-dydx","email":"ken@dydx.exchange"},{"name":"luka-arcus","email":"luka@dydx.exchange"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/da-reader_0.2.0_1785981976679_0.2524529326686493"},"_hasShrinkwrap":false}},"time":{"created":"2026-08-05T23:16:13.733Z","modified":"2026-08-06T02:06:17.235Z","0.1.0":"2026-08-05T23:16:14.066Z","0.1.1":"2026-08-05T23:38:45.983Z","0.2.0":"2026-08-06T02:06:16.824Z"},"bugs":{"url":"https://github.com/arcus-xyz/da-reader/issues"},"license":"Apache-2.0","homepage":"https://github.com/arcus-xyz/da-reader#readme","keywords":["perps","data-availability","sbe","s3","indexer"],"repository":{"type":"git","url":"git+https://github.com/arcus-xyz/da-reader.git"},"description":"Read and decode perps-chain Data Availability records (S3 block-inputs): SigV4 fetch, SBE fill decode, market units, real-time tip following.","maintainers":[{"name":"k-dydx","email":"ken@dydx.exchange"},{"name":"luka-arcus","email":"luka@dydx.exchange"}],"readme":"# da-reader\n\nRead and decode the perps chain's **Data Availability records** from TypeScript, in real\ntime. Zero runtime dependencies.\n\nThe chain's DA layer is an S3 bucket of `block-inputs/<n>` objects (gzipped JSON wrapping\nSBE binary frames), where `n` is exactly the on-chain block height (genesis = 0).\nThis package handles the whole read path:\n\n| layer | module | what it does |\n|---|---|---|\n| storage | `S3DaReader` | SigV4-signed GET/HEAD (no @aws-sdk), gunzip, JSON |\n| frames | `readHeader`, `decodeFillLeg`, `decodeTransferOp`, … | SBE decode of fill legs (template 3) and transfers (template 60) |\n| blocks | `processBlockInputs`, `iterateFrames`, … | wrapper walk, oracle prices, per-block stats |\n| stream | `followBlockInputs`, `findTip` | ordered, gap-free, self-healing real-time tail |\n| units | `PERP_MARKETS`, `priceTicksToUsd`, … | market dimension + exact unit conversions |\n\nFrame offsets are append-only-stable: the protocol's schema rules only ever add fields\nat the tail, so a decoder pinned to today's offsets keeps working as the schema grows.\n\n## Quickstart\n\n```ts\nimport {\n  S3DaReader, s3DaConfigFromEnv, followBlockInputs,\n  marketSymbol, priceTicksToUsd, quoteQuantumsToUsd, sizeQuantumsToBase,\n} from \"@arcus-xyz/da-reader\";\n\nconst reader = new S3DaReader(s3DaConfigFromEnv()); // DA_S3_* env vars (see .env.example)\n\nfor await (const { blockNumber, block } of followBlockInputs(reader, { start: \"tip\" })) {\n  for (const f of block.fills) {\n    console.log(\n      `block ${blockNumber}: ${marketSymbol(f.marketId)} ` +\n      `${sizeQuantumsToBase(f.fillSize, f.marketId)} @ $${priceTicksToUsd(f.fillPrice, f.marketId)} ` +\n      `($${quoteQuantumsToUsd(f.notionalQq).toFixed(2)})`,\n    );\n  }\n}\n```\n\nRunnable versions: [`examples/print-fills.ts`](examples/print-fills.ts) (real-time\nprinter) and [`examples/indexer.ts`](examples/indexer.ts) (durable-cursor indexer\nskeleton — the pattern for a service that persists blocks).\n\n## Following in real time\n\n`followBlockInputs(source, opts)` is an async generator that yields\n`{ blockNumber, atTip, block }` **strictly in order, never skipping a block**:\n\n- **start**: `number` (first block yielded), `\"tip\"` (resolve the current tip via\n  `findTip`, yield it first), or `{ after: n }` — resume from a persisted cursor\n  verbatim; the +1 lives in the library so there is no off-by-one to remember.\n- **Catch-up** fetches `concurrency` blocks in parallel (default 8); a 404 marks the tip\n  (`onCaughtUp` fires once), after which it polls every `pollIntervalMs` (default 3s)\n  and drains bursts without sleeping.\n- **Errors are classified** (see below): transient ones retry forever with capped\n  jittered backoff (`onError` sees every attempt), fatal ones throw out of the\n  generator. Corrupt bodies (bad gzip) get `corruptRetries` refetches.\n- **Shutdown**: pass an `AbortSignal`; the generator returns cleanly mid-anything.\n  Breaking out of a `for await` loop also closes it.\n- **Gap alarm**: if the tail stalls on a missing block while later blocks exist (a\n  writer failure — should never happen), `onGap` fires after `stallProbeAfterMs`\n  (default 2 min) so it alarms instead of looking like a quiet chain. The stream still\n  never skips.\n\nDurable-cursor pattern: persist `blockNumber` after processing each block (ideally in\nthe same transaction as your writes), and restart with `start: { after: persisted }`.\nRe-processing a block after a crash-before-persist is possible — make writes idempotent\nper `(blockNumber, fill.msgIndex)`.\n\n## Error taxonomy\n\n| error | meaning | follower behavior |\n|---|---|---|\n| `null` return | 404 — block doesn't exist (the tip) | switch to tip polling |\n| `S3RequestError` `.isTransient` (5xx, 429, SlowDown) | S3 blip | retry with backoff, forever |\n| network `TypeError` / `TimeoutError` | connection trouble | retry with backoff, forever |\n| `S3RequestError` 403/400/301 | bad creds / wrong region — never self-heals | throw immediately |\n| `CorruptObjectError` | body failed gunzip/JSON (truncated read?) | bounded refetch, then throw |\n| decode throws (bad schemaId, truncated frame) | schema violation | throw immediately — fail loud |\n\n`S3RequestError` carries `.status` and `.awsCode` (parsed from the S3 XML error body).\nClock skew is self-healing: on `RequestTimeTooSkewed` the reader learns the server\noffset and re-signs once.\n\n## Credentials\n\nRead-only credentials for the DA bucket, via `s3DaConfigFromEnv()` (`DA_S3_*`, see\n[.env.example](.env.example)) or an explicit `S3DaConfig`. No bucket coordinates are\nbaked into the package — supply the bucket, prefix, and region you read from.\nSTS/temporary credentials are supported via `sessionToken`.\n\n⚠️ The IAM policy should grant **`s3:GetObject` and `s3:ListBucket`**. Without\n`ListBucket`, S3 answers **403 (not 404) for missing keys**, which breaks \"404 = tip\"\n— the follower would treat every tip poll as a fatal error.\n\n## Units (exact math)\n\nWire integers are `bigint` and stay exact; convert at the display boundary only:\n\n| wire value | unit | to display |\n|---|---|---|\n| `fillPrice`, oracle prices | ticks | `priceTicksToUsd(ticks, marketId)` |\n| `fillSize` | quantums | `sizeQuantumsToBase(q, marketId)` |\n| `fee`, `notionalQq`, `closedPnl`, `netQuoteBalance` | quote quantums | `quoteQuantumsToUsd(qq)` (= /1e9) |\n\nPer-fill notional = `fillSize × fillPrice` in quote quantums, exactly, for every market\n(the wire invariant `tickSize × quantumSize = 1e-9`). Do **not** use the frame's\n`filledNotional` field for this — it is cumulative per order.\n\n**Bigint gotcha**: `JSON.stringify` throws on bigints, so raw `ProcessedBlock`s can't go\nstraight into a log line or HTTP body. Use `processedBlockToJson` / `fillToJson` /\n`blockStatsToJson` (bigints → lossless decimal strings).\n\n## Aggregating fills correctly\n\nA taker crossing N maker orders emits **one aggregated taker leg + N maker legs**, and\n`tradeId` is not a taker↔maker join key. So: count trades and sum volume from **maker\nlegs** (`role === ROLE.MAKER`), sum fees over **all legs**, and treat\n`deriveBlockStats` as the reference implementation of those rules. Addresses come out\nlowercase-hex; EIP-55 checksumming is left to consumers (needs keccak, which we don't\ndepend on).\n\n## Deposits and withdrawals (template 60)\n\n`AccountTransferUpdateOperation` is the only frame carrying money into or out of an\naccount. `decodeTransferOp(frame)` decodes one; pair it with `iterateFrames`, which\n`processBlockInputs` does not filter for you:\n\n```ts\nimport { iterateFrames, decodeTransferOp, TEMPLATE, quoteQuantumsToUsd } from \"@arcus-xyz/da-reader\";\n\nfor (const slot of iterateFrames(rawBlockJson)) {\n  if (slot.templateId !== TEMPLATE.ACCOUNT_TRANSFER_UPDATE_OPERATION || slot.buf === null) continue;\n  const op = decodeTransferOp(slot.buf);\n  if (op.movement === null) continue; // op.movementSkipped says why\n  const { kind, account, subaccount, amountQq, netQuoteBalanceQq } = op.movement;\n  console.log(\n    `${kind} $${quoteQuantumsToUsd(amountQq)} -> ${account}/${subaccount}` +\n    ` (balance now $${quoteQuantumsToUsd(netQuoteBalanceQq)})` +\n    (op.hasRootchainProvenance ? ` [rootchain inbox #${op.rootchainQueueIndex}]` : \"\"),\n  );\n}\n```\n\n`op` is the faithful decode — both account slots, both balances, `status`, sequence\nnumbers, provenance. `op.movement` is the derived one-account view, non-null **only** for\nan applied deposit or withdrawal; `op.movementSkipped` names the case when it is null\n(`rejected`, `not_a_credit`, `unreadable_account`, `party_mismatch`,\n`non_positive_amount`). Three traps this handles that a hand-rolled parser typically\ndoes not:\n\n- **`status` is not decoration.** Ten rejection codes exist, and when `status !== 0` the\n  operation was *not applied* — its balance and sequence fields describe **pre-op** state.\n  Crediting one debits an account the exchange never debited. `movement` is null for\n  those; `transferStatusName(op.status)` names the rejection for a log line.\n- **There are two account slots.** A deposit is chain→account, a withdrawal is\n  account→chain, so which slot holds the trading account — and which of the two\n  `netQuoteBalance` fields is *its* balance — depends on the operation type. The other\n  slot holds the `ROBINHOOD_CHAIN` sentinel (`source`/`destination` report it as\n  `{ kind: \"chain\" }`); a frame where it is missing is one we have misidentified, and\n  yields `party_mismatch` rather than a plausible-looking amount.\n- **`rootchainQueueIndex` cannot signal its own absence.** The rootchain inbox is\n  zero-indexed, so `0` is a valid slot. Provenance comes from a non-zero\n  `rootchainPayloadHash`; the index is therefore `null` unless `hasRootchainProvenance`.\n\n`netQuoteBalanceQq` is the account's authoritative post-op balance, stamped at apply time\n— the same class of signal as a fill leg's `netQuoteBalance`. Resync to it rather than\naccumulating `±amountQq`.\n\nOffsets are derived from the protocol schema and checked against frames produced by the\nexchange's own reference encoder ([`tests/t60GoldenFrames.ts`](tests/t60GoldenFrames.ts)), so\nthe offset table is tested against something other than our own reading of it.\n\nRunnable version: [`examples/print-transfers.ts`](examples/print-transfers.ts).\n\n## Decoding other templates\n\n`processBlockInputs` surfaces fills (template 3). Block-inputs also carry templates 23,\n28, 31 (PositionUpdate), 32 (FundingTick) and 36. To decode those without forking the\nwrapper-walking logic:\n\n```ts\nfor (const slot of iterateFrames(rawBlockJson)) {\n  if (slot.templateId === TEMPLATE.FUNDING_TICK) {\n    // slot.buf is the full frame (64-byte header + body); bring your own offsets\n  }\n}\n```\n\n`readIdString`, `decodeAccountIdAt` and `isChainSentinelAt` read a `char[36]` `idString`\nslot at any offset, which is most of what a new template needs.\n\n## Development\n\n```\nnpm install\nnpm test          # node:test via tsx; fixtures are synthetic frames\nnpm run typecheck\nnpm run build     # tsup -> dist (ESM + d.ts)\n```\n\nThe package is ESM-only with a Node **22.12** floor — that version can `require()` an\nESM graph, so CommonJS consumers can `require(\"@arcus-xyz/da-reader\")` without a separate\nCJS build. Installing straight from the git repo works too; a `prepare` script builds\n`dist/` on install.\n\nLicensed under [Apache-2.0](LICENSE).\n","readmeFilename":"README.md"}