{"_id":"@avandar/etl","_rev":"3-bb1a0a9da5bb58b8c0586bfbb581fac1","name":"@avandar/etl","dist-tags":{"latest":"0.1.2"},"versions":{"0.1.0":{"name":"@avandar/etl","version":"0.1.0","license":"MIT","_id":"@avandar/etl@0.1.0","maintainers":[{"name":"jpsyx","email":"pablowritescode@gmail.com"}],"homepage":"https://github.com/AvandarLabs/avandar/tree/main/packages/node/etl#readme","bugs":{"url":"https://github.com/AvandarLabs/avandar/issues"},"dist":{"shasum":"dc40da65694cab06212c965859af3f5de1475515","tarball":"https://registry.npmjs.org/@avandar/etl/-/etl-0.1.0.tgz","fileCount":6,"integrity":"sha512-WmFGFIbr+JY4vwsGSCVsfDf17xol3JUSIX4VH4d1v8migz5kwEfVwLi5D/w2HaOWlfQkxlTmUl5Fj6FWjfd+gQ==","signatures":[{"sig":"MEUCIQDGCa2PAo0jgg9sgb53IdanyZ6kKTPx3b2WpCKVwMCz8gIgJimMuOWUPQL7M1bY7fBBDEAVOAU39O0ye4FVuNIWC6k=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@avandar%2fetl@0.1.0","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"unpackedSize":86287},"type":"module","_from":"file:avandar-etl-0.1.0.tgz","exports":{".":{"types":"./dist/index.d.ts","default":"./dist/index.js"}},"scripts":{"test":"vitest run","build":"tsup","test:watch":"vitest","type-check":"tsc --noEmit"},"_npmUser":{"name":"jpsyx","email":"pablowritescode@gmail.com"},"_resolved":"/tmp/65eae8fd0aef73bdddf9d4ade8ac19fe/avandar-etl-0.1.0.tgz","_integrity":"sha512-WmFGFIbr+JY4vwsGSCVsfDf17xol3JUSIX4VH4d1v8migz5kwEfVwLi5D/w2HaOWlfQkxlTmUl5Fj6FWjfd+gQ==","repository":{"url":"git+https://github.com/AvandarLabs/avandar.git","type":"git","directory":"packages/node/etl"},"_npmVersion":"11.12.1","description":"Avandar ETL library","directories":{},"sideEffects":false,"_nodeVersion":"24.15.0","dependencies":{"@avandar/utils":"0.1.0","@avandar/modules":"0.1.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","duckdb":"^1.4.4","vitest":"^3.2.4","typescript":"~5.9.3","@supabase/supabase-js":"^2.98.0"},"peerDependencies":{"duckdb":"^1.4.4","@supabase/supabase-js":"^2.98.0"},"_npmOperationalInternal":{"tmp":"tmp/etl_0.1.0_1786411752044_0.5955957295330185","host":"s3://npm-registry-packages-npm-production"}},"0.1.1":{"name":"@avandar/etl","version":"0.1.1","license":"MIT","_id":"@avandar/etl@0.1.1","maintainers":[{"name":"jpsyx","email":"pablowritescode@gmail.com"}],"homepage":"https://github.com/AvandarLabs/avandar/tree/main/packages/node/etl#readme","bugs":{"url":"https://github.com/AvandarLabs/avandar/issues"},"dist":{"shasum":"161387882cdc845e5e37fb1fd66c1ef1331d2ec6","tarball":"https://registry.npmjs.org/@avandar/etl/-/etl-0.1.1.tgz","fileCount":6,"integrity":"sha512-DOcriOKpnhBmiYiDZ1+DvhywHx8jXJU2fuPDaXKWcdKyFONUBPqUNeH3s9yIKUHAY6u6d9ulzcodV7pOqJ8kPA==","signatures":[{"sig":"MEUCIEx+SjAdRSwAgpTNDvZA0w/+8rB1+nZBsRz0336lgFIrAiEA+JWnydVW0v3tsGbV2ZsUFwTHG6xJQ/pqvN3um3wN6JU=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@avandar%2fetl@0.1.1","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"unpackedSize":86287},"type":"module","_from":"file:avandar-etl-0.1.1.tgz","exports":{".":{"types":"./dist/index.d.ts","default":"./dist/index.js"}},"scripts":{"test":"vitest run","build":"tsup","test:watch":"vitest","type-check":"tsc --noEmit"},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:0dfaad4c-9755-4120-935a-5a27a1cc8cac"}},"_resolved":"/tmp/55e08a5a58e303634a3327942dd81ef0/avandar-etl-0.1.1.tgz","_integrity":"sha512-DOcriOKpnhBmiYiDZ1+DvhywHx8jXJU2fuPDaXKWcdKyFONUBPqUNeH3s9yIKUHAY6u6d9ulzcodV7pOqJ8kPA==","repository":{"url":"git+https://github.com/AvandarLabs/avandar.git","type":"git","directory":"packages/node/etl"},"_npmVersion":"11.12.1","description":"Avandar ETL library","directories":{},"sideEffects":false,"_nodeVersion":"24.15.0","dependencies":{"@avandar/utils":"0.1.1","@avandar/modules":"0.1.1"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","duckdb":"^1.4.4","vitest":"^3.2.4","typescript":"~5.9.3","@supabase/supabase-js":"^2.98.0"},"peerDependencies":{"duckdb":"^1.4.4","@supabase/supabase-js":"^2.98.0"},"_npmOperationalInternal":{"tmp":"tmp/etl_0.1.1_1786447314966_0.9411530391734046","host":"s3://npm-registry-packages-npm-production"}},"0.1.2":{"name":"@avandar/etl","version":"0.1.2","description":"Avandar ETL library","license":"MIT","repository":{"type":"git","url":"git+https://github.com/AvandarLabs/avandar.git","directory":"packages/node/etl"},"homepage":"https://github.com/AvandarLabs/avandar/tree/main/packages/node/etl#readme","bugs":{"url":"https://github.com/AvandarLabs/avandar/issues"},"type":"module","exports":{".":{"types":"./dist/index.d.ts","default":"./dist/index.js"}},"sideEffects":false,"dependencies":{"@avandar/modules":"0.1.2","@avandar/utils":"0.1.2"},"devDependencies":{"@supabase/supabase-js":"^2.98.0","duckdb":"^1.4.4","tsup":"^8.5.0","typescript":"~5.9.3","vitest":"^3.2.4"},"publishConfig":{"access":"public"},"peerDependencies":{"@supabase/supabase-js":"^2.98.0","duckdb":"^1.4.4"},"scripts":{"build":"tsup","test":"vitest run","test:watch":"vitest","type-check":"tsc --noEmit"},"_id":"@avandar/etl@0.1.2","_integrity":"sha512-k/BEpX9VTmUDGQ0Dvui3L2KdPQyUkj99Hx5/S0ODT54dLXuZjVQAfnZKSt2vaTkUedbNAhReuENGktObxalJyQ==","_resolved":"/tmp/089e3cab588dc3a1fd4b8abb959a6167/avandar-etl-0.1.2.tgz","_from":"file:avandar-etl-0.1.2.tgz","_nodeVersion":"24.15.0","_npmVersion":"11.12.1","dist":{"integrity":"sha512-k/BEpX9VTmUDGQ0Dvui3L2KdPQyUkj99Hx5/S0ODT54dLXuZjVQAfnZKSt2vaTkUedbNAhReuENGktObxalJyQ==","shasum":"64664275d8074fc454949505c322724a97a4cd2b","tarball":"https://registry.npmjs.org/@avandar/etl/-/etl-0.1.2.tgz","fileCount":6,"unpackedSize":86287,"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@avandar%2fetl@0.1.2","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEYCIQD+4Ut4iS53W6c56EvTtfuPycVAwmXF7R1mfIGkKbbVxwIhAMzR3EA8fzXdrlIU7ivBTX02md2EHDOhmevKbAp1/ncq"}]},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:0dfaad4c-9755-4120-935a-5a27a1cc8cac"}},"directories":{},"maintainers":[{"name":"jpsyx","email":"pablowritescode@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/etl_0.1.2_1786500349463_0.7008630217970333"},"_hasShrinkwrap":false}},"time":{"created":"2026-08-11T01:29:11.902Z","modified":"2026-08-12T02:05:50.077Z","0.1.0":"2026-08-11T01:29:12.278Z","0.1.1":"2026-08-11T11:21:55.118Z","0.1.2":"2026-08-12T02:05:49.646Z"},"bugs":{"url":"https://github.com/AvandarLabs/avandar/issues"},"license":"MIT","homepage":"https://github.com/AvandarLabs/avandar/tree/main/packages/node/etl#readme","repository":{"type":"git","url":"git+https://github.com/AvandarLabs/avandar.git","directory":"packages/node/etl"},"description":"Avandar ETL library","maintainers":[{"name":"jpsyx","email":"pablowritescode@gmail.com"}],"readme":"# @avandar/etl\n\nAvandar ETL library. Tools for building Node.js Extract → Transform → Load\npipelines whose intermediate output is CSV on disk and whose final output\nis ZSTD Parquet (locally and, optionally, in Supabase Storage).\n\nThe library is built around two units:\n\n- **`EtlEngine`** — a module factory that runs a 3-step pipeline against a\n  conventional directory layout under `etl-input/` and `etl-output/`.\n- **`NodeDuckDb`** — a thin wrapper around the `duckdb` Node bindings used\n  internally by the engine for CSV sniffing, view creation, and Parquet\n  export.\n\nESM only. Requires Node 22+.\n\n## Install\n\n```sh\npnpm add @avandar/etl\npnpm add duckdb @supabase/supabase-js\n```\n\nBoth are peer dependencies. `duckdb` is a native module with a real compile\nstep, and you hold its connections, so it must be a single copy you control.\n`@supabase/supabase-js` is a peer for the same reason: you pass a live client\nacross the API boundary.\n\n## Usage\n\n```ts\nimport { EtlEngine } from \"@avandar/etl\";\n\nconst pipeline = EtlEngine.create({\n  name: \"world-bank__wdi\",\n  extract: async ({ pipelineRunId }) => {\n    await EtlEngine.storeExtractedData({\n      pipelineName: \"world-bank__wdi\",\n      pipelineRunId,\n      sourcePath: \"./WDIData.csv\",\n      destinationBasename: \"WDIData.csv\",\n    });\n    return { files: [{ name: \"WDIData.csv\", mimeType: \"text/csv\" }] };\n  },\n  transform: () => [{\n    name: \"wdi\",\n    columns: [/* ... DuckDbSniffableDataType per column ... */],\n  }],\n  load: async ({ pipelineName, pipelineRunId, parquetTableBaseNames }) => {\n    await EtlEngine.uploadParquetToStorage({\n      pipelineName,\n      pipelineRunId,\n      parquetTableBaseNames,\n    });\n  },\n});\n\nawait pipeline.run();\n```\n\nThe engine runs `extract` → `transform` → CSV-to-Parquet conversion →\n`load`, materialising intermediate files under\n`etl-output/<pipeline-name>/<pipelineRunId>/{extract,transform,load}/`.\n\n---\n\n## EtlEngine\n\n`EtlEngine` is a module factory; call `EtlEngine.create({...})` to get a\nrunnable pipeline. The factory also carries a few static helpers as\nproperties.\n\n### `EtlEngine.create(state)` (factory)\n\nState fields:\n\n| Field       | Description                                                                                  |\n| ----------- | -------------------------------------------------------------------------------------------- |\n| `name`      | Pipeline name (used as the folder segment under `etl-output/`)                               |\n| `extract`   | `({ pipelineRunId }) => Promise<ExtractedDataContext>` — writes CSVs to the extract directory |\n| `transform` | `(extracted, { pipelineRunId }) => TransformedDataDescriptionForParquet[]` — describes the transformed CSVs the engine should turn into Parquet |\n| `load`      | `({ pipelineName, pipelineRunId, parquetTableBaseNames }) => Promise<void>` — sinks Parquet files |\n\nThe returned pipeline module exposes `.run()`, which executes the full\nsequence and returns `{ pipelineRunId }`.\n\n### Static helpers\n\n| Function                          | Description                                                                              |\n| --------------------------------- | ---------------------------------------------------------------------------------------- |\n| `EtlEngine.storeExtractedData`    | Copies a source file into `etl-output/<pipeline>/<run>/extract/<destinationBasename>`    |\n| `EtlEngine.getLoadParquetPathForTable` | Returns the absolute path for a load-stage Parquet file                              |\n| `EtlEngine.uploadParquetToStorage`| Uploads every load-stage Parquet to Supabase Storage at `{bucket}/{pipeline}/datasets/<table>.parquet` (uses `SUPABASE_URL` and `SUPABASE_SERVICE_ROLE_KEY`; bucket from `SUPABASE_OPENDATA_BUCKET`, default `opendata`) |\n\n### Paths and configuration\n\n| Export                          | Description                                                                  |\n| ------------------------------- | ---------------------------------------------------------------------------- |\n| `ETL_INPUT_BASE_DIR`            | Constant: `\"etl-input\"`                                                      |\n| `ETL_OUTPUT_BASE_DIR`           | Constant: `\"etl-output\"`                                                     |\n| `ETL_PATHS_ROOT_ENV`            | Env var name read for the absolute paths root (`ETL_PATHS_ROOT`)             |\n| `getEtlPipelineInputDir`        | `etl-input/<pipeline-name>/`                                                 |\n| `getEtlInputDir`                | Extract output dir for a run (`etl-output/.../extract`)                      |\n| `getEtlOutputDir`               | Any-stage output dir for a run (`extract` or `transform`)                    |\n| `getEtlLoadDir`                 | Load output dir for a run (`etl-output/.../load`)                            |\n| `setEtlPathsRootForTesting`     | Pin the filesystem root used by all path helpers (tests only)                |\n| `resetEtlPathsRootForTesting`   | Clear the testing override (call in `afterEach`)                             |\n\n### Transform-step types\n\n| Export                              | Description                                                            |\n| ----------------------------------- | ---------------------------------------------------------------------- |\n| `transformedCsvsToParquetBlobs`     | Reads `<transformDir>/<name>.csv` for each description, returns ZSTD Parquet `Blob`s |\n| `TransformedColumnDescription`      | `{ name, type: DuckDbSniffableDataType }` for a single column          |\n| `TransformedDataDescriptionForParquet` | `{ name, columns: TransformedColumnDescription[] }` for one table   |\n\nIf `columns` is empty, the engine relies on DuckDB CSV auto-detection;\notherwise each column must specify a sniffable type for explicit casts.\n\n---\n\n## NodeDuckDb\n\nA thin Node.js wrapper around the `duckdb` native bindings. Use directly\nwhen you need raw DuckDB access outside of an `EtlEngine` pipeline.\n\n### `class NodeDuckDb`\n\nConstructor: `new NodeDuckDb({ databasePath? })`. Defaults to `:memory:`.\n\n| Method                                       | Description                                                                  |\n| -------------------------------------------- | ---------------------------------------------------------------------------- |\n| `runRawQuery(sql, { params? })`              | Runs a query and returns normalized row objects. `$name$` placeholders are substituted with `String(value)` |\n| `execSQL(sql)`                               | Runs DDL/COPY/etc. without returning rows                                    |\n| `sniffCsv({ csvPath })`                      | Returns `[{ name, type }]` per column using DuckDB `sniff_csv`               |\n| `readCsvIntoView(options)`                   | `CREATE OR REPLACE VIEW` over a `read_csv(...)` call                         |\n| `exportTableOrViewAsZSTDParquetBlob(name)`   | Writes the table/view to a temp Parquet file with ZSTD, returns the bytes    |\n| `summarizeParquetFile(parquetPath)`          | Returns `{ rowCount, columnNames, columnTypeDescriptions }` for a Parquet file |\n| `close()`                                    | Closes connection and database handle                                        |\n\n### CSV types\n\n| Export                            | Description                                                              |\n| --------------------------------- | ------------------------------------------------------------------------ |\n| `NodeDuckDbReadCsvColumn`         | `{ name, type: DuckDbSniffableDataType }` — column spec for `read_csv`   |\n| `NodeDuckDbSniffCsvColumn`        | `{ name, type: DuckDbSniffableDataType }` — returned by `sniffCsv`       |\n| `NodeDuckDbReadCsvIntoViewOptions`| Options accepted by `readCsvIntoView` (path, view name, columns, header, skip, delimiter, autoDetect) |\n\n### DuckDB sniffable types\n\n| Export                                | Description                                                                                  |\n| ------------------------------------- | -------------------------------------------------------------------------------------------- |\n| `DuckDbSniffableDataType`             | Union of the persisted CSV-sniff types: `BOOLEAN`, `BIGINT`, `DOUBLE`, `TIME`, `DATE`, `TIMESTAMP`, `VARCHAR` |\n| `duckDbDescribeColumnTypeToSniffable` | Maps a DuckDB `DESCRIBE` `column_type` string to a `DuckDbSniffableDataType`                  |\n| `SNIFF_CSV_MAX_ROWS`                  | Sample size passed to `sniff_csv` (`10_000`)                                                  |\n\n## License\n\nMIT\n","readmeFilename":"README.md"}