{"_id":"@backloghq/termlog","_rev":"5-600bcd3b39601a70c912c88fa9df0d6c","name":"@backloghq/termlog","dist-tags":{"latest":"0.2.0"},"versions":{"0.1.0":{"name":"@backloghq/termlog","version":"0.1.0","keywords":["full-text-search","fts","inverted-index","bm25","lsm","embedded","search-engine"],"author":{"name":"mbocevski"},"license":"MIT","_id":"@backloghq/termlog@0.1.0","maintainers":[{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"}],"homepage":"https://github.com/backloghq/termlog#readme","bugs":{"url":"https://github.com/backloghq/termlog/issues"},"dist":{"shasum":"18d6d942447b6cae78c28181fa6c35540958fed1","tarball":"https://registry.npmjs.org/@backloghq/termlog/-/termlog-0.1.0.tgz","fileCount":52,"integrity":"sha512-Z4Wq4s7S0ZCkAD6GCCC4E4ZZF4PWvo6er/tXv0WyHyr7ocXuvf1QKyi9WnQIwIRmJrlax2BpJ34paVPfZdRLHg==","signatures":[{"sig":"MEUCICN/hdvb9NAdlnLRn/QwyV2Qkbe02bhUQlAaM5AENVmSAiEAoEpxTaks7aOrRhH9H2VH5lMjP0ksoz5IfnfDhnQPjow=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":219080},"main":"dist/index.js","type":"module","types":"dist/index.d.ts","engines":{"node":">=22"},"exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"gitHead":"d6016eca85ef4706c687059af0bf19b5e4b22cf9","scripts":{"dev":"tsc --watch","lint":"eslint src/ tests/","test":"vitest run","build":"tsc","test:watch":"vitest","test:coverage":"vitest run --coverage","prepublishOnly":"npm run build"},"_npmUser":{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"},"repository":{"url":"git+https://github.com/backloghq/termlog.git","type":"git"},"_npmVersion":"11.13.0","description":"Log-structured full-text search index — segment-based posting lists with LSM compaction, BM25 ranking, zero native dependencies.","directories":{},"sideEffects":false,"_nodeVersion":"25.9.0","_hasShrinkwrap":false,"devDependencies":{"eslint":"^10.0.0","vitest":"^4.1.2","@eslint/js":"^10.0.0","typescript":"~6.0.2","@types/node":"^25.0.0","typescript-eslint":"^8.58.1","@vitest/coverage-v8":"^4.1.2"},"_npmOperationalInternal":{"tmp":"tmp/termlog_0.1.0_1777964774407_0.51103708784905","host":"s3://npm-registry-packages-npm-production"}},"0.1.1":{"name":"@backloghq/termlog","version":"0.1.1","keywords":["full-text-search","fts","inverted-index","bm25","lsm","embedded","search-engine"],"author":{"name":"mbocevski"},"license":"MIT","_id":"@backloghq/termlog@0.1.1","maintainers":[{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"}],"homepage":"https://github.com/backloghq/termlog#readme","bugs":{"url":"https://github.com/backloghq/termlog/issues"},"dist":{"shasum":"fea168ead5f11780d094b749188cad98d97fc9f2","tarball":"https://registry.npmjs.org/@backloghq/termlog/-/termlog-0.1.1.tgz","fileCount":52,"integrity":"sha512-vsthAHlfEmSXfyzsE/NDayLrnWPqPoXbHJ0uVvTtGks6I51wrb5IfHWNkf0D94//r9YuVMygQjD8LXOSqRNsjA==","signatures":[{"sig":"MEYCIQCUBltTIHaen8HVqU8EFwzW2/iRONKxkt2ZvVg17H2akwIhAIVUaHZSs76J3Un83JDyoyCTrMEosXWHNXcy5G2HY7Hk","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":225310},"main":"dist/index.js","type":"module","types":"dist/index.d.ts","engines":{"node":">=22"},"exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"gitHead":"0a4e918a336d1af4aa9233d9273e72a920f92c9c","scripts":{"dev":"tsc --watch","lint":"eslint src/ tests/","test":"vitest run","build":"tsc","test:watch":"vitest","test:coverage":"vitest run --coverage","prepublishOnly":"npm run build"},"_npmUser":{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"},"deprecated":"Docs-only republish — use 0.1.2 (the 0.1.1 tarball shipped with a stale CHANGELOG date; code identical)","repository":{"url":"git+https://github.com/backloghq/termlog.git","type":"git"},"_npmVersion":"11.13.0","description":"Log-structured full-text search index — segment-based posting lists with LSM compaction, BM25 ranking, zero native dependencies.","directories":{},"sideEffects":false,"_nodeVersion":"25.9.0","_hasShrinkwrap":false,"devDependencies":{"eslint":"^10.0.0","vitest":"^4.1.2","@eslint/js":"^10.0.0","typescript":"~6.0.2","@types/node":"^25.0.0","typescript-eslint":"^8.58.1","@vitest/coverage-v8":"^4.1.2"},"_npmOperationalInternal":{"tmp":"tmp/termlog_0.1.1_1777967274790_0.03666429715505948","host":"s3://npm-registry-packages-npm-production"}},"0.1.2":{"name":"@backloghq/termlog","version":"0.1.2","keywords":["full-text-search","fts","inverted-index","bm25","lsm","embedded","search-engine"],"author":{"name":"mbocevski"},"license":"MIT","_id":"@backloghq/termlog@0.1.2","maintainers":[{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"}],"homepage":"https://github.com/backloghq/termlog#readme","bugs":{"url":"https://github.com/backloghq/termlog/issues"},"dist":{"shasum":"32bbae9303561d86a89a829d754901351e863e6f","tarball":"https://registry.npmjs.org/@backloghq/termlog/-/termlog-0.1.2.tgz","fileCount":52,"integrity":"sha512-Jjn8bbF6l9KJmkSHbQcW5UzYKnTKjtw4v9GaEuAQjNNTvOWtZrnCA9CoTZbNT41zRfHdsjotbSLtDnwxbj4FGw==","signatures":[{"sig":"MEYCIQD9Eh7GczUgJvgpUNwQX+Rm2/pFWdqfZWgAsF9QLHNhegIhAIpU1hOalDlohD1w7rHBHreIH40wGZd4QqBA4BxV9yUj","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":225551},"main":"dist/index.js","type":"module","types":"dist/index.d.ts","engines":{"node":">=22"},"exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"gitHead":"dfde54355ce9e803e4d476e94549b493dd92e93d","scripts":{"dev":"tsc --watch","lint":"eslint src/ tests/","test":"vitest run","build":"tsc","test:watch":"vitest","test:coverage":"vitest run --coverage","prepublishOnly":"npm run build"},"_npmUser":{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"},"repository":{"url":"git+https://github.com/backloghq/termlog.git","type":"git"},"_npmVersion":"11.13.0","description":"Log-structured full-text search index — segment-based posting lists with LSM compaction, BM25 ranking, zero native dependencies.","directories":{},"sideEffects":false,"_nodeVersion":"25.9.0","_hasShrinkwrap":false,"devDependencies":{"eslint":"^10.0.0","vitest":"^4.1.2","@eslint/js":"^10.0.0","typescript":"~6.0.2","@types/node":"^25.0.0","typescript-eslint":"^8.58.1","@vitest/coverage-v8":"^4.1.2"},"_npmOperationalInternal":{"tmp":"tmp/termlog_0.1.2_1777967922974_0.07666791647034499","host":"s3://npm-registry-packages-npm-production"}},"0.2.0":{"name":"@backloghq/termlog","version":"0.2.0","description":"Log-structured full-text search index — segment-based posting lists with LSM compaction, BM25 ranking, zero native dependencies.","main":"dist/index.js","types":"dist/index.d.ts","type":"module","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"scripts":{"build":"tsc","dev":"tsc --watch","lint":"eslint src/ tests/","test":"vitest run","test:watch":"vitest","test:coverage":"vitest run --coverage","prepublishOnly":"npm run build"},"keywords":["full-text-search","fts","inverted-index","bm25","lsm","embedded","search-engine"],"author":{"name":"mbocevski"},"license":"MIT","sideEffects":false,"repository":{"type":"git","url":"git+https://github.com/backloghq/termlog.git"},"bugs":{"url":"https://github.com/backloghq/termlog/issues"},"homepage":"https://github.com/backloghq/termlog#readme","engines":{"node":">=22"},"devDependencies":{"@eslint/js":"^10.0.0","@types/node":"^25.0.0","@vitest/coverage-v8":"^4.1.2","eslint":"^10.0.0","typescript":"~6.0.2","typescript-eslint":"^8.58.1","vitest":"^4.1.2"},"gitHead":"36bfcc7d859eb82f3bad6aeab1adff98104c1f1c","_id":"@backloghq/termlog@0.2.0","_nodeVersion":"26.1.0","_npmVersion":"11.14.1","dist":{"integrity":"sha512-9Aw3n5nseVGnqkfQ7a7nVda2vHEjP+PJx/tJulX3PW4N4WS3Y6Dj0/H76Femo0WEvnP7eaf+be2dS2muqfKC4g==","shasum":"45e7d1dbec46fee2974a353b4015a0e4fa1c4dfb","tarball":"https://registry.npmjs.org/@backloghq/termlog/-/termlog-0.2.0.tgz","fileCount":56,"unpackedSize":288318,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIFGlXz6t65sBZeu4Ae4RlH87OhvTDZtqxJuB10gZja7aAiEA8RtHM/oIBSkU+PmqisJ702xmWWNf/TrfxJkBCe3b7tM="}]},"_npmUser":{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"},"directories":{},"maintainers":[{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/termlog_0.2.0_1780144251388_0.6445124175293089"},"_hasShrinkwrap":false}},"time":{"created":"2026-05-05T07:06:14.302Z","modified":"2026-05-30T12:30:51.696Z","0.1.0":"2026-05-05T07:06:14.568Z","0.1.1":"2026-05-05T07:47:54.955Z","0.1.2":"2026-05-05T07:58:43.138Z","0.2.0":"2026-05-30T12:30:51.554Z"},"bugs":{"url":"https://github.com/backloghq/termlog/issues"},"author":{"name":"mbocevski"},"license":"MIT","homepage":"https://github.com/backloghq/termlog#readme","keywords":["full-text-search","fts","inverted-index","bm25","lsm","embedded","search-engine"],"repository":{"type":"git","url":"git+https://github.com/backloghq/termlog.git"},"description":"Log-structured full-text search index — segment-based posting lists with LSM compaction, BM25 ranking, zero native dependencies.","maintainers":[{"name":"fenrirbaest","email":"marko.bocevski@gmail.com"}],"readme":"# @backloghq/termlog\n\nLog-structured full-text search index — segment-based posting lists with LSM compaction, BM25 ranking, zero native dependencies.\n\n## Install\n\n```\nnpm install @backloghq/termlog\n```\n\n## Usage\n\n```ts\nimport { TermLog } from \"@backloghq/termlog\";\n\nconst index = await TermLog.open({ dir: \"./my-index\" });\n\nawait index.add(\"doc-1\", \"the quick brown fox\");\nawait index.add(\"doc-2\", \"the lazy dog\");\nawait index.flush();\n\nconst results = await index.search(\"fox\", { limit: 10 });\n// [{ docId: \"doc-1\", score: 0.655... }]  (BM25 — exact value depends on corpus)\n\nawait index.remove(\"doc-1\");\nawait index.close();\n```\n\n## Why\n\nExisting FTS engines (Lucene, Tantivy) require native deps or a JVM. Most pure-JS alternatives serialize the index to a single in-memory blob — fine for small corpora, but they hit per-file size cliffs in the tens of thousands of documents. Termlog uses immutable on-disk segments with LSM compaction so the corpus scales without those ceilings.\n\n## Architecture\n\n- **Posting lists** — `term → [docId, tf]`, compressed with VByte / delta encoding. (Positions reserved for a future release.)\n- **Term dictionary** — sorted on disk; binary search for lookup.\n- **Segments** — self-contained immutable files (term dict + postings). New writes create a new segment. Compaction merges N segments into 1.\n- **Query execution** — boolean (AND/OR) via posting iterators (zigzag merge for AND, union scan for OR), BM25 scoring on top.\n- **Storage** — abstracted via `StorageBackend`; local FS by default, S3 via [@backloghq/termlog-s3](https://github.com/backloghq/termlog-s3).\n\n## S3 backend\n\nS3 support is provided by the companion package [@backloghq/termlog-s3](https://github.com/backloghq/termlog-s3):\n\n```bash\nnpm install @backloghq/termlog @backloghq/termlog-s3\n```\n\n```ts\nimport { TermLog } from \"@backloghq/termlog\";\nimport { S3Backend } from \"@backloghq/termlog-s3\";\nimport { S3Client } from \"@aws-sdk/client-s3\";\n\nconst index = await TermLog.open({\n  dir: \"my-index\",\n  backend: new S3Backend({\n    client: new S3Client({ region: \"us-east-1\" }),\n    bucket: \"my-bucket\",\n    prefix: \"my-index/\",\n  }),\n});\n```\n\nSee the [termlog-s3 README](https://github.com/backloghq/termlog-s3) for IAM permissions, lifecycle rules, and MinIO/LocalStack usage.\n\n## Options\n\n| Option | Default | Description |\n|---|---|---|\n| `fanout` | 4 | Same-tier segment count that triggers a merge (size-tiered LSM) |\n| `flushThreshold` | 1000 | Docs in write buffer before auto-flush |\n| `k1` | 1.2 | BM25 term-saturation parameter |\n| `b` | 0.75 | BM25 length-normalization parameter |\n\n## Errors\n\n| Class | When thrown |\n|---|---|\n| `ManifestCorruptionError` | manifest.json contains invalid JSON |\n| `ManifestVersionError` | manifest version is outside the supported range |\n| `SegmentCorruptionError` | CRC32 mismatch or missing segment file (`.region` tells you which) |\n| `MappingCorruptionError` | docids.snap or docids.log is corrupt |\n| `TokenizerMismatchError` | reopening an index with a different tokenizer config |\n| `IndexLockedError` | another process holds the advisory `.lock` file |\n| `WriteStreamError` | base class for streaming write failures (S3 multipart, etc.) |\n\n## Stats\n\n| Method | Returns | Description |\n|---|---|---|\n| `docCount()` | `number` | Documents indexed across all flushed segments |\n| `segmentCount()` | `number` | Number of active on-disk segments |\n| `estimatedBytes()` | `number` | Approximate in-memory footprint (postings buffers + sidecar arrays + Maps); lower-bound estimate for memory-budget callers |\n\n## Multi-writer / S3 safety\n\nTermlog is **single-writer per index directory**. On local FS an advisory `.lock` file prevents concurrent opens in the same process group. On S3 (or any shared storage) there is no distributed lock — you must ensure at most one writer per index path.\n\n## License\n\nMIT\n","readmeFilename":"README.md"}